ltcai 11.2.0 → 11.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. package/README.md +46 -53
  2. package/docs/CHANGELOG.md +61 -0
  3. package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
  4. package/docs/DEVELOPMENT.md +1 -1
  5. package/docs/MULTI_AGENT_RUNTIME.md +1 -1
  6. package/docs/ONBOARDING.md +1 -1
  7. package/docs/OPERATIONS.md +6 -2
  8. package/docs/PERMISSION_MODE.md +1 -1
  9. package/docs/TRUST_MODEL.md +1 -1
  10. package/docs/WHY_LATTICE.md +1 -1
  11. package/docs/kg-schema.md +2 -2
  12. package/docs/v11.3.0_PLAN.md +202 -0
  13. package/docs/v11.4.0_RUST_FOUNDATION_PLAN.md +176 -0
  14. package/lattice_brain/__init__.py +1 -1
  15. package/lattice_brain/graph/_kg_common/__init__.py +287 -0
  16. package/lattice_brain/graph/_kg_common/extraction.py +516 -0
  17. package/lattice_brain/graph/_kg_common/relations.py +161 -0
  18. package/lattice_brain/graph/_kg_common/text.py +479 -0
  19. package/lattice_brain/graph/discovery_index/__init__.py +35 -0
  20. package/lattice_brain/graph/discovery_index/cleanup.py +182 -0
  21. package/lattice_brain/graph/discovery_index/extract.py +137 -0
  22. package/lattice_brain/graph/discovery_index/scan.py +411 -0
  23. package/lattice_brain/graph/discovery_index/upsert.py +495 -0
  24. package/lattice_brain/graph/projection/__init__.py +42 -0
  25. package/lattice_brain/graph/projection/curation.py +500 -0
  26. package/lattice_brain/graph/{projection.py → projection/v2_schema.py} +15 -477
  27. package/lattice_brain/graph/retrieval/__init__.py +54 -0
  28. package/lattice_brain/graph/retrieval/context.py +197 -0
  29. package/lattice_brain/graph/retrieval/graph_view.py +319 -0
  30. package/lattice_brain/graph/retrieval/hybrid.py +488 -0
  31. package/lattice_brain/graph/retrieval/maintenance.py +121 -0
  32. package/lattice_brain/graph/retrieval/signals.py +95 -0
  33. package/lattice_brain/graph/retrieval_vector/__init__.py +42 -0
  34. package/lattice_brain/graph/retrieval_vector/fingerprint.py +97 -0
  35. package/lattice_brain/graph/retrieval_vector/indexing.py +347 -0
  36. package/lattice_brain/graph/retrieval_vector/search.py +560 -0
  37. package/lattice_brain/graph/retrieval_vector/status.py +374 -0
  38. package/lattice_brain/ingestion/__init__.py +130 -0
  39. package/lattice_brain/ingestion/_contract.py +90 -0
  40. package/lattice_brain/ingestion/constants.py +127 -0
  41. package/lattice_brain/ingestion/folder_scan.py +57 -0
  42. package/lattice_brain/ingestion/folders.py +258 -0
  43. package/lattice_brain/ingestion/hashing.py +26 -0
  44. package/lattice_brain/ingestion/jobs_api.py +107 -0
  45. package/lattice_brain/ingestion/models.py +80 -0
  46. package/lattice_brain/ingestion/pipeline.py +486 -0
  47. package/lattice_brain/ingestion/quality.py +209 -0
  48. package/lattice_brain/ingestion/routing.py +295 -0
  49. package/lattice_brain/multimodal/__init__.py +164 -0
  50. package/lattice_brain/multimodal/audio.py +77 -0
  51. package/lattice_brain/multimodal/common.py +118 -0
  52. package/lattice_brain/multimodal/images.py +498 -0
  53. package/lattice_brain/multimodal/ports.py +169 -0
  54. package/lattice_brain/multimodal/video.py +410 -0
  55. package/lattice_brain/portability/__init__.py +90 -0
  56. package/lattice_brain/portability/_contract.py +42 -0
  57. package/lattice_brain/portability/backups.py +338 -0
  58. package/lattice_brain/portability/bundles.py +136 -0
  59. package/lattice_brain/portability/constants.py +93 -0
  60. package/lattice_brain/portability/fsops.py +138 -0
  61. package/lattice_brain/portability/service.py +41 -0
  62. package/lattice_brain/{portability.py → portability/sharing.py} +44 -677
  63. package/lattice_brain/runtime/__init__.py +1 -1
  64. package/lattice_brain/runtime/multi_agent.py +1 -1
  65. package/latticeai/__init__.py +1 -1
  66. package/latticeai/api/chronicle.py +63 -0
  67. package/latticeai/core/agent/__init__.py +93 -0
  68. package/latticeai/core/agent/_contract.py +79 -0
  69. package/latticeai/core/agent/context.py +57 -0
  70. package/latticeai/core/agent/deps.py +125 -0
  71. package/latticeai/core/agent/execution.py +622 -0
  72. package/latticeai/core/agent/planning.py +145 -0
  73. package/latticeai/core/agent/recovery.py +157 -0
  74. package/latticeai/core/agent/runtime.py +210 -0
  75. package/latticeai/core/agent/verification.py +231 -0
  76. package/latticeai/core/embedding_providers/__init__.py +151 -0
  77. package/latticeai/core/embedding_providers/base.py +199 -0
  78. package/latticeai/core/embedding_providers/captions.py +162 -0
  79. package/latticeai/core/embedding_providers/profiles.py +126 -0
  80. package/latticeai/core/embedding_providers/text.py +350 -0
  81. package/latticeai/core/embedding_providers/vision.py +352 -0
  82. package/latticeai/core/file_generation/__init__.py +115 -0
  83. package/latticeai/core/file_generation/bundles.py +76 -0
  84. package/latticeai/core/file_generation/extraction.py +154 -0
  85. package/latticeai/core/file_generation/inference.py +235 -0
  86. package/latticeai/core/file_generation/orchestration.py +152 -0
  87. package/latticeai/core/file_generation/prompting.py +117 -0
  88. package/latticeai/core/file_generation/repair.py +114 -0
  89. package/latticeai/core/file_generation/sanitize.py +61 -0
  90. package/latticeai/core/file_generation/validation.py +201 -0
  91. package/latticeai/core/legacy_compatibility.py +1 -1
  92. package/latticeai/core/marketplace.py +1 -1
  93. package/latticeai/core/messages.py +9 -0
  94. package/latticeai/core/workspace_os_constants.py +1 -1
  95. package/latticeai/integrations/telegram_bot/__init__.py +123 -0
  96. package/latticeai/integrations/telegram_bot/__main__.py +17 -0
  97. package/latticeai/integrations/telegram_bot/config.py +86 -0
  98. package/latticeai/integrations/telegram_bot/dispatch.py +311 -0
  99. package/latticeai/integrations/telegram_bot/flows.py +478 -0
  100. package/latticeai/integrations/telegram_bot/helpers.py +322 -0
  101. package/latticeai/integrations/telegram_bot/screens.py +394 -0
  102. package/latticeai/models/router/__init__.py +88 -0
  103. package/latticeai/models/router/_contract.py +66 -0
  104. package/latticeai/models/router/branding.py +56 -0
  105. package/latticeai/models/router/catalog.py +69 -0
  106. package/latticeai/models/router/documents.py +199 -0
  107. package/latticeai/models/router/errors.py +37 -0
  108. package/latticeai/models/router/generation.py +258 -0
  109. package/latticeai/models/router/loading.py +291 -0
  110. package/latticeai/models/router/local_models.py +85 -0
  111. package/latticeai/models/router/registry.py +147 -0
  112. package/latticeai/runtime/build_phases/__init__.py +82 -0
  113. package/latticeai/runtime/build_phases/features.py +407 -0
  114. package/latticeai/runtime/build_phases/foundation.py +555 -0
  115. package/latticeai/runtime/build_phases/web.py +492 -0
  116. package/latticeai/runtime/runtime_context.py +1 -0
  117. package/latticeai/services/architecture_readiness.py +48 -19
  118. package/latticeai/services/brain_intelligence/__init__.py +58 -0
  119. package/latticeai/services/brain_intelligence/_contract.py +71 -0
  120. package/latticeai/services/brain_intelligence/consistency.py +193 -0
  121. package/latticeai/services/brain_intelligence/constants.py +47 -0
  122. package/latticeai/services/brain_intelligence/digest.py +258 -0
  123. package/latticeai/services/brain_intelligence/health.py +331 -0
  124. package/latticeai/services/brain_intelligence/proposals.py +264 -0
  125. package/latticeai/services/brain_intelligence/sampling.py +84 -0
  126. package/latticeai/services/brain_intelligence/service.py +48 -0
  127. package/latticeai/services/chronicle.py +557 -0
  128. package/latticeai/services/memory_service/__init__.py +52 -0
  129. package/latticeai/services/memory_service/_contract.py +100 -0
  130. package/latticeai/services/memory_service/brief.py +431 -0
  131. package/latticeai/services/memory_service/constants.py +57 -0
  132. package/latticeai/services/memory_service/maintenance.py +138 -0
  133. package/latticeai/services/memory_service/manager.py +186 -0
  134. package/latticeai/services/memory_service/proof.py +136 -0
  135. package/latticeai/services/memory_service/recall.py +225 -0
  136. package/latticeai/services/memory_service/service.py +48 -0
  137. package/latticeai/services/memory_service/stores.py +110 -0
  138. package/latticeai/services/model_runtime/__init__.py +322 -0
  139. package/latticeai/services/model_runtime/cloud.py +87 -0
  140. package/latticeai/services/model_runtime/download.py +282 -0
  141. package/latticeai/services/model_runtime/engines.py +341 -0
  142. package/latticeai/services/model_runtime/loading.py +178 -0
  143. package/latticeai/services/model_runtime/service.py +129 -0
  144. package/latticeai/services/model_runtime/state.py +131 -0
  145. package/latticeai/services/model_runtime/status.py +255 -0
  146. package/latticeai/services/product_readiness.py +15 -7
  147. package/latticeai/setup/wizard/__init__.py +126 -0
  148. package/latticeai/setup/wizard/catalog.py +172 -0
  149. package/latticeai/setup/wizard/detect.py +323 -0
  150. package/latticeai/setup/wizard/install.py +348 -0
  151. package/latticeai/setup/wizard/paths.py +168 -0
  152. package/latticeai/setup/wizard/plans.py +74 -0
  153. package/latticeai/setup/wizard/recommend.py +320 -0
  154. package/package.json +6 -2
  155. package/scripts/bump_version.py +14 -0
  156. package/scripts/capture_release_evidence.mjs +33 -21
  157. package/scripts/check_current_release_docs.mjs +1 -1
  158. package/scripts/check_i18n_namespace_coverage.mjs +41 -4
  159. package/scripts/check_max_file_lines.mjs +102 -0
  160. package/scripts/check_release_evidence_bound.mjs +30 -15
  161. package/scripts/check_screenshot_pixel_delta.py +34 -4
  162. package/scripts/check_server_i18n.mjs +1 -0
  163. package/scripts/generate_rust_parity_fixtures.py +562 -0
  164. package/scripts/lib/mock_server_fingerprint.mjs +94 -0
  165. package/scripts/release_screen_claims.json +31 -2
  166. package/src-tauri/Cargo.lock +361 -3
  167. package/src-tauri/Cargo.toml +6 -1
  168. package/src-tauri/src/backend.rs +349 -0
  169. package/src-tauri/src/folder.rs +33 -0
  170. package/src-tauri/src/main.rs +97 -399
  171. package/src-tauri/tauri.conf.json +1 -1
  172. package/static/app/asset-manifest.json +41 -37
  173. package/static/app/assets/Act-yYpYnn0v.js +1 -0
  174. package/static/app/assets/AdminConsole-DL3Cr5pL.js +1 -0
  175. package/static/app/assets/{Brain-tuhI4sOC.js → Brain-C1HBN0Wf.js} +2 -2
  176. package/static/app/assets/BrainHome-DoXRhUUC.js +2 -0
  177. package/static/app/assets/BrainSignals-6yR6ir5t.js +1 -0
  178. package/static/app/assets/Capture-CFIRsFNE.js +1 -0
  179. package/static/app/assets/Chronicle-BZbEgiwN.js +1 -0
  180. package/static/app/assets/CommandPalette-D2pMxC2I.js +1 -0
  181. package/static/app/assets/Library-DwO3yZST.js +1 -0
  182. package/static/app/assets/{LivingBrain-DBwhto14.js → LivingBrain-Jn1GK0-S.js} +1 -1
  183. package/static/app/assets/ProductFlow-B-w1R4Oo.js +1 -0
  184. package/static/app/assets/ReviewCard-6B27X8Vg.js +3 -0
  185. package/static/app/assets/System-DW8F-2xL.js +1 -0
  186. package/static/app/assets/arrow-left-DXvKg9U6.js +1 -0
  187. package/static/app/assets/{bot-Cia42c2h.js → bot-IM_E_Y12.js} +1 -1
  188. package/static/app/assets/brain-Ci1CkWjM.js +1 -0
  189. package/static/app/assets/{button-2j2Ijzgq.js → button-COwyqfHM.js} +1 -1
  190. package/static/app/assets/circle-check-DfInj-qD.js +1 -0
  191. package/static/app/assets/{circle-pause-BEFeWpVW.js → circle-pause-DEM4A1Y5.js} +1 -1
  192. package/static/app/assets/{circle-play-ujXMcHxl.js → circle-play-C9djDuLd.js} +1 -1
  193. package/static/app/assets/{cpu-k4awryFq.js → cpu-DFdo1gw-.js} +1 -1
  194. package/static/app/assets/{download-DFbLJ_ig.js → download-SnJL6oqk.js} +1 -1
  195. package/static/app/assets/{folder-open-7y_b6xkM.js → folder-open-CqZeDkjE.js} +1 -1
  196. package/static/app/assets/{hard-drive-Bidh02Kr.js → hard-drive-j1jJXYYf.js} +1 -1
  197. package/static/app/assets/{index-DwDl9-8Y.css → index-BLPb5lmE.css} +1 -1
  198. package/static/app/assets/index-_u5iUHDr.js +10 -0
  199. package/static/app/assets/input-B0lPdRQZ.js +1 -0
  200. package/static/app/assets/link-2-CoFbooHS.js +1 -0
  201. package/static/app/assets/{permissionCopy-Bpb83Hx9.js → permissionCopy-BsyLxtao.js} +1 -1
  202. package/static/app/assets/primitives-DEbN-d6p.js +1 -0
  203. package/static/app/assets/search-BybIWPNd.js +1 -0
  204. package/static/app/assets/{share-2-BH1M-WNi.js → share-2-CVtZ_ewX.js} +1 -1
  205. package/static/app/assets/{shield-alert-BlKdBXcG.js → shield-alert-CBi2GNWM.js} +1 -1
  206. package/static/app/assets/{textarea-CCWbUfFB.js → textarea-DNMpB5ih.js} +1 -1
  207. package/static/app/assets/{useFocusTrap-YdHQ7pJ1.js → useFocusTrap-C83t3GXF.js} +1 -1
  208. package/static/app/assets/useMutation-DtbJDoyz.js +1 -0
  209. package/static/app/assets/{useQuery-CXQiwbVT.js → useQuery-Dcp1OChy.js} +1 -1
  210. package/static/app/assets/utils-BlZr7Pd4.js +4 -0
  211. package/static/app/assets/workspace-jJY4RuAV.js +1 -0
  212. package/static/app/index.html +4 -4
  213. package/static/sw.js +1 -1
  214. package/lattice_brain/graph/_kg_common.py +0 -1331
  215. package/lattice_brain/graph/discovery_index.py +0 -1141
  216. package/lattice_brain/graph/retrieval.py +0 -1120
  217. package/lattice_brain/graph/retrieval_vector.py +0 -1293
  218. package/lattice_brain/ingestion.py +0 -1525
  219. package/lattice_brain/multimodal.py +0 -1258
  220. package/latticeai/core/agent.py +0 -1465
  221. package/latticeai/core/embedding_providers.py +0 -1196
  222. package/latticeai/core/file_generation.py +0 -1047
  223. package/latticeai/integrations/telegram_bot.py +0 -1390
  224. package/latticeai/models/router.py +0 -1007
  225. package/latticeai/runtime/build_phases.py +0 -1450
  226. package/latticeai/services/brain_intelligence.py +0 -1083
  227. package/latticeai/services/memory_service.py +0 -1177
  228. package/latticeai/services/model_runtime.py +0 -1281
  229. package/latticeai/setup/wizard.py +0 -1310
  230. package/static/app/assets/Act-AWf0SAKp.js +0 -1
  231. package/static/app/assets/AdminConsole-D0u8Tiyj.js +0 -1
  232. package/static/app/assets/BrainHome-Ts7G_Ila.js +0 -2
  233. package/static/app/assets/BrainSignals-jMYgQ2Ar.js +0 -1
  234. package/static/app/assets/Capture-CqOSzyPr.js +0 -1
  235. package/static/app/assets/CommandPalette-DC0Bzh-I.js +0 -1
  236. package/static/app/assets/Library-CX-bbhmK.js +0 -1
  237. package/static/app/assets/ProductFlow-BHA2cfKI.js +0 -1
  238. package/static/app/assets/ReviewCard-BUhCKRNM.js +0 -3
  239. package/static/app/assets/System-Bu2t5hn1.js +0 -1
  240. package/static/app/assets/arrow-left-Dzwa5zRb.js +0 -1
  241. package/static/app/assets/brain-DJMoqrwx.js +0 -1
  242. package/static/app/assets/index-BpYkzcVm.js +0 -10
  243. package/static/app/assets/input-DSlJJxRs.js +0 -1
  244. package/static/app/assets/primitives-BCx6TvfG.js +0 -1
  245. package/static/app/assets/search-Cgy8cCFJ.js +0 -1
  246. package/static/app/assets/utils-zqPZJxdx.js +0 -4
  247. package/static/app/assets/workspace-DXTihhfU.js +0 -1
@@ -0,0 +1,77 @@
1
+ """A recording as a memory — transcribed through the injected port, or not.
2
+
3
+ The whole module is three small pieces because that is all an audio memory is:
4
+ the recording's facts, one attempt at turning it into words, and an honest
5
+ score for how much of it a typed question can reach afterwards. Without a
6
+ transcriber the memory is still kept and simply says so.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass, field
12
+ from typing import Any, Dict, List, Optional
13
+
14
+ from .ports import MultimodalPorts
15
+
16
+
17
+ @dataclass
18
+ class AudioFacts:
19
+ """A recording, its transcript, and how honestly we got one."""
20
+
21
+ path: str
22
+ #: ``ok`` | ``unavailable`` | ``failed`` | ``supplied``
23
+ transcription_status: str = "unavailable"
24
+ transcript: str = ""
25
+ detail: str = ""
26
+ segments: List[Dict[str, Any]] = field(default_factory=list)
27
+
28
+ @property
29
+ def searchable(self) -> bool:
30
+ return bool(self.transcript)
31
+
32
+
33
+ def transcribe_audio(
34
+ path: str,
35
+ *,
36
+ ports: Optional[MultimodalPorts] = None,
37
+ transcript: Optional[str] = None,
38
+ ) -> AudioFacts:
39
+ """Transcribe a recording through the injected port, or say why not.
40
+
41
+ ``transcript`` lets a caller that already has text (a phone's own
42
+ dictation, ``VoiceCaptureService``) skip the local model entirely. An
43
+ absent transcriber yields ``transcription_status="unavailable"`` and an
44
+ empty transcript — the recording is still remembered by title and path,
45
+ and the result never claims it is searchable.
46
+ """
47
+ ports = ports or MultimodalPorts()
48
+ supplied = str(transcript or "").strip()
49
+ if supplied:
50
+ return AudioFacts(path=str(path), transcription_status="supplied", transcript=supplied)
51
+ if ports.transcriber is None:
52
+ return AudioFacts(
53
+ path=str(path),
54
+ transcription_status="unavailable",
55
+ detail="no local transcriber is configured",
56
+ )
57
+ try:
58
+ text = str(ports.transcriber(str(path)) or "").strip()
59
+ except Exception as exc: # noqa: BLE001 — a broken transcriber is a state
60
+ return AudioFacts(path=str(path), transcription_status="failed", detail=str(exc))
61
+ if not text:
62
+ return AudioFacts(
63
+ path=str(path),
64
+ transcription_status="failed",
65
+ detail="the transcriber returned no text",
66
+ )
67
+ return AudioFacts(path=str(path), transcription_status="ok", transcript=text)
68
+
69
+
70
+ def audio_quality_score(facts: AudioFacts) -> Dict[str, Any]:
71
+ """How much of this recording is actually retrievable later."""
72
+ if not facts.transcript:
73
+ return {"score": 0.0, "reasons": ["no_transcript"]}
74
+ # A transcript is text: length is the only honest extra signal here, and
75
+ # the text pipeline scores the wording itself downstream.
76
+ score = 0.5 + 0.5 * min(1.0, len(facts.transcript) / 400.0)
77
+ return {"score": round(score, 4), "reasons": ["transcript"]}
@@ -0,0 +1,118 @@
1
+ """Modality taxonomy, size budgets, and the pure helpers every door shares.
2
+
3
+ The tables here are the module's own decisions (``.mp4`` is video unless the
4
+ capture surface says otherwise), deliberately kept ahead of the platform's
5
+ ``mimetypes`` answer. Nothing in this file touches a model, a decoder, or the
6
+ graph, which is what lets images, audio and video all import it without
7
+ importing each other.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import mimetypes
14
+ from pathlib import Path
15
+ from typing import List, Optional
16
+
17
+ # ── modality taxonomy ────────────────────────────────────────────────────────
18
+ MODALITY_TEXT = "text"
19
+ MODALITY_IMAGE = "image"
20
+ MODALITY_AUDIO = "audio"
21
+ MODALITY_VIDEO = "video"
22
+
23
+ IMAGE_EXTENSIONS = frozenset(
24
+ {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic"}
25
+ )
26
+ # Containers that are *only* ever audio. ``.mp4``/``.webm`` are deliberately
27
+ # absent: by extension alone they are video, and a voice memo recorded in one
28
+ # of them arrives through an explicit audio MIME type (or through
29
+ # ``VoiceCaptureService``, where the user already said "this is a memo").
30
+ # ``.mid``/``.midi`` are listed for a second reason: CPython's *built-in* mime
31
+ # table has neither, so ``mimetypes`` answers "audio/midi" only on a host that
32
+ # ships a system mime file (macOS reads /etc/apache2/mime.types; a slim Linux
33
+ # container has nothing). Leaving them to the fallback let the platform decide
34
+ # what a MIDI file is — a module table exists precisely so it does not.
35
+ AUDIO_EXTENSIONS = frozenset(
36
+ {".m4a", ".mp3", ".wav", ".aac", ".flac", ".ogg", ".opus", ".mid", ".midi"}
37
+ )
38
+ VIDEO_EXTENSIONS = frozenset({".mp4", ".webm", ".mov", ".mkv", ".avi", ".m4v"})
39
+ #: Subtitle/caption files a video may arrive with. Same basename, so a
40
+ #: ``standup.mp4`` next to a ``standup.srt`` is one memory, not two.
41
+ SUBTITLE_EXTENSIONS = ("srt", "vtt")
42
+
43
+ #: Why a video is recognized and still refused — surfaced to the caller. In
44
+ #: 11.1.0 the reason was *scope* (nothing was implemented). Since 11.2.0 the
45
+ #: implementation exists and the only remaining reason is a **runtime** one:
46
+ #: this machine has no ``ffmpeg``, and inventing frames is not an option.
47
+ VIDEO_UNAVAILABLE_DETAIL = (
48
+ "video ingestion needs ffmpeg on this machine and none was found; the file "
49
+ "was not stored (install ffmpeg to enable keyframe extraction)"
50
+ )
51
+ #: Kept under its 11.1.0 name so existing importers keep working; the reason it
52
+ #: carries has changed from "out of scope" to "unavailable on this machine".
53
+ VIDEO_OUT_OF_SCOPE = VIDEO_UNAVAILABLE_DETAIL
54
+
55
+ #: Longest OCR/caption body kept on the node (a screenshot is not a novel).
56
+ MAX_INDEX_TEXT_CHARS = 20_000
57
+ #: Summary column budget, matching every other ingest door in the graph.
58
+ SUMMARY_CHARS = 500
59
+ #: Fixed-width chunking for OCR bodies that outgrow the summary.
60
+ IMAGE_CHUNK_CHARS = 900
61
+ #: Longest edge of the stored thumbnail, in pixels.
62
+ THUMBNAIL_EDGE = 96
63
+ #: A thumbnail is a UI affordance, not an archive — drop it past this size.
64
+ MAX_THUMBNAIL_CHARS = 24_000
65
+
66
+
67
+ def detect_modality(
68
+ path: Optional[str] = None, mime_type: Optional[str] = None
69
+ ) -> str:
70
+ """``text`` | ``image`` | ``audio`` | ``video`` for one candidate file.
71
+
72
+ The declared MIME type wins when it carries a usable top-level type: the
73
+ capture surface saw the bytes, this function only sees a name. Otherwise
74
+ the extension decides, and an unknown extension is ``text`` so existing
75
+ behaviour is untouched.
76
+ """
77
+ declared = str(mime_type or "").strip().lower().split(";")[0].split("/")[0]
78
+ if declared in {MODALITY_IMAGE, MODALITY_AUDIO, MODALITY_VIDEO}:
79
+ return declared
80
+ # The extension tables come before ``mimetypes`` on purpose: they are where
81
+ # this module's decisions live (``.mp4`` is video unless someone who saw
82
+ # the bytes says otherwise), and the stdlib table varies by platform.
83
+ suffix = Path(str(path or "")).suffix.lower()
84
+ if suffix in IMAGE_EXTENSIONS:
85
+ return MODALITY_IMAGE
86
+ if suffix in AUDIO_EXTENSIONS:
87
+ return MODALITY_AUDIO
88
+ if suffix in VIDEO_EXTENSIONS:
89
+ return MODALITY_VIDEO
90
+ if path:
91
+ guessed, _ = mimetypes.guess_type(str(path))
92
+ top = str(guessed or "").split("/")[0]
93
+ if top in {MODALITY_IMAGE, MODALITY_AUDIO, MODALITY_VIDEO}:
94
+ return top
95
+ return MODALITY_TEXT
96
+
97
+
98
+ def _sha256_file(path: Path) -> str:
99
+ digest = hashlib.sha256()
100
+ with path.open("rb") as handle:
101
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
102
+ digest.update(block)
103
+ return digest.hexdigest()
104
+
105
+
106
+ def _sha256_text(text: str) -> str:
107
+ return hashlib.sha256(str(text).encode("utf-8", "ignore")).hexdigest()
108
+
109
+
110
+ def _split_index_text(text: str) -> List[str]:
111
+ """Fixed-width split for OCR bodies — no markdown or code structure here."""
112
+ body = str(text or "").strip()
113
+ if len(body) <= SUMMARY_CHARS:
114
+ return []
115
+ return [
116
+ body[start : start + IMAGE_CHUNK_CHARS]
117
+ for start in range(0, len(body), IMAGE_CHUNK_CHARS)
118
+ ]
@@ -0,0 +1,498 @@
1
+ """A picture as a first-class memory: what was observed, and what was written.
2
+
3
+ Two halves, in order: :func:`extract_image_facts` observes one file (size, OCR,
4
+ caption, vector — each with its own status), and :func:`write_image_memory`
5
+ turns those facts into an ``Image`` node with its chunks, concepts and source
6
+ link. Video reuses both verbatim for its keyframes, which is why nothing here
7
+ knows what a video is.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import base64
13
+ import io
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Any, Callable, Dict, List, Optional
17
+
18
+ from ..quiet import quiet
19
+ from ..utils import utc_now_iso
20
+ from .common import (
21
+ MAX_INDEX_TEXT_CHARS,
22
+ MAX_THUMBNAIL_CHARS,
23
+ MODALITY_IMAGE,
24
+ SUMMARY_CHARS,
25
+ THUMBNAIL_EDGE,
26
+ _sha256_file,
27
+ _sha256_text,
28
+ _split_index_text,
29
+ )
30
+ from .ports import MultimodalPorts
31
+
32
+
33
+ @dataclass
34
+ class ImageFacts:
35
+ """Everything observed about one image, and the status of each attempt."""
36
+
37
+ path: str
38
+ width: Optional[int] = None
39
+ height: Optional[int] = None
40
+ image_format: Optional[str] = None
41
+ mode: Optional[str] = None
42
+ #: ``ok`` | ``empty`` | ``unavailable`` | ``failed`` | ``skipped``
43
+ ocr_status: str = "skipped"
44
+ ocr_text: str = ""
45
+ ocr_detail: str = ""
46
+ #: ``ok`` | ``unavailable``
47
+ caption_status: str = "unavailable"
48
+ caption: Optional[str] = None
49
+ #: ``ok`` | ``unavailable`` | ``failed``
50
+ embedding_status: str = "unavailable"
51
+ embedding: Optional[List[float]] = None
52
+ embedding_detail: str = ""
53
+ thumbnail: Optional[str] = None
54
+ #: Set when the file could not be opened as an image at all.
55
+ error: str = ""
56
+
57
+ @property
58
+ def readable(self) -> bool:
59
+ return not self.error
60
+
61
+ def index_text(self) -> str:
62
+ """The text a search engine can actually match this image on."""
63
+ parts = [part for part in (self.caption, self.ocr_text) if part]
64
+ return "\n".join(parts).strip()[:MAX_INDEX_TEXT_CHARS]
65
+
66
+ def as_metadata(self) -> Dict[str, Any]:
67
+ """Flat, JSON-safe view stored on the graph node."""
68
+ payload: Dict[str, Any] = {
69
+ "modality": MODALITY_IMAGE,
70
+ "width": self.width,
71
+ "height": self.height,
72
+ "format": self.image_format,
73
+ "mode": self.mode,
74
+ "ocr_status": self.ocr_status,
75
+ "ocr_chars": len(self.ocr_text),
76
+ "caption_status": self.caption_status,
77
+ "vision_embedding": self.embedding_status,
78
+ }
79
+ if self.ocr_text:
80
+ payload["ocr_text"] = self.ocr_text
81
+ if self.ocr_detail:
82
+ payload["ocr_detail"] = self.ocr_detail
83
+ if self.caption:
84
+ payload["caption"] = self.caption
85
+ if self.embedding_detail:
86
+ payload["vision_embedding_detail"] = self.embedding_detail
87
+ if self.thumbnail:
88
+ payload["thumbnail"] = self.thumbnail
89
+ if self.error:
90
+ payload["image_error"] = self.error
91
+ return payload
92
+
93
+
94
+ def _open_image(path: str) -> Any:
95
+ """Pillow's ``Image.open`` behind a guarded import."""
96
+ from PIL import Image # local import: keeps the module importable without it
97
+
98
+ return Image.open(str(path))
99
+
100
+
101
+ def _thumbnail_data_uri(image: Any, edge: int = THUMBNAIL_EDGE) -> Optional[str]:
102
+ """A tiny inline PNG the Evidence panel can render with no new route.
103
+
104
+ Serving the original file would mean either a new static route over the
105
+ user's disk or reusing ``/local/serve``, which exists precisely to make
106
+ every read pass an explicit approval. A 96px data URI on the node dodges
107
+ both: it is already inside the graph the user is looking at.
108
+ """
109
+ try:
110
+ small = image.copy()
111
+ small.thumbnail((edge, edge))
112
+ if small.mode not in {"RGB", "L"}:
113
+ small = small.convert("RGB")
114
+ buffer = io.BytesIO()
115
+ small.save(buffer, format="PNG")
116
+ except Exception: # noqa: BLE001 — a missing thumbnail is not a failed ingest
117
+ quiet()
118
+ return None
119
+ encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
120
+ if len(encoded) > MAX_THUMBNAIL_CHARS:
121
+ return None
122
+ return f"data:image/png;base64,{encoded}"
123
+
124
+
125
+ def _run_ocr(image: Any) -> Dict[str, str]:
126
+ """OCR through ``pytesseract`` when it is installed, honestly otherwise."""
127
+ try:
128
+ import pytesseract # optional local binary + wrapper
129
+ except Exception as exc: # noqa: BLE001 — absence is a state, not an error
130
+ return {"status": "unavailable", "text": "", "detail": str(exc)}
131
+ try:
132
+ text = str(pytesseract.image_to_string(image) or "").strip()
133
+ except Exception as exc: # noqa: BLE001 — a broken OCR runtime is a state
134
+ return {"status": "failed", "text": "", "detail": str(exc)}
135
+ if not text:
136
+ return {"status": "empty", "text": "", "detail": "no text found in the image"}
137
+ return {"status": "ok", "text": text[:MAX_INDEX_TEXT_CHARS], "detail": ""}
138
+
139
+
140
+ def extract_image_facts(
141
+ path: str,
142
+ *,
143
+ ports: Optional[MultimodalPorts] = None,
144
+ ocr: bool = True,
145
+ thumbnail: bool = True,
146
+ ) -> ImageFacts:
147
+ """Observe one image: size, OCR, caption, vector — each with its status.
148
+
149
+ Never raises. An unreadable file returns ``ImageFacts(error=...)`` so the
150
+ caller can record "we saw this file and could not read it" instead of
151
+ losing the memory entirely.
152
+ """
153
+ ports = ports or MultimodalPorts()
154
+ facts = ImageFacts(path=str(path))
155
+ try:
156
+ with _open_image(path) as image:
157
+ facts.width = int(image.width)
158
+ facts.height = int(image.height)
159
+ facts.image_format = image.format
160
+ facts.mode = image.mode
161
+ if ocr:
162
+ result = _run_ocr(image)
163
+ facts.ocr_status = result["status"]
164
+ facts.ocr_text = result["text"]
165
+ facts.ocr_detail = result["detail"]
166
+ if thumbnail:
167
+ facts.thumbnail = _thumbnail_data_uri(image)
168
+ except Exception as exc: # noqa: BLE001 — an unreadable image is a state
169
+ facts.error = str(exc)
170
+ return facts
171
+
172
+ if ports.captioner is not None:
173
+ caption = _safe_caption(ports.captioner, facts.path)
174
+ if caption:
175
+ facts.caption = caption
176
+ facts.caption_status = "ok"
177
+
178
+ if ports.vision_embedder is not None:
179
+ _apply_vision_embedding(facts, ports.vision_embedder)
180
+ return facts
181
+
182
+
183
+ def _safe_caption(
184
+ captioner: Callable[[str], Optional[str]], path: str
185
+ ) -> Optional[str]:
186
+ """Ask the VLM; a failure means *no caption*, never an invented one."""
187
+ try:
188
+ caption = captioner(path)
189
+ except Exception: # noqa: BLE001 — a broken captioner must not fail an ingest
190
+ quiet()
191
+ return None
192
+ cleaned = str(caption or "").strip()
193
+ return cleaned or None
194
+
195
+
196
+ def _apply_vision_embedding(
197
+ facts: ImageFacts, embedder: Callable[[str], List[float]]
198
+ ) -> None:
199
+ try:
200
+ vector = [float(value) for value in embedder(facts.path)]
201
+ except Exception as exc: # noqa: BLE001 — an absent model is not a failed ingest
202
+ facts.embedding_status = "failed"
203
+ facts.embedding_detail = str(exc)
204
+ return
205
+ if not vector:
206
+ facts.embedding_status = "failed"
207
+ facts.embedding_detail = "vision provider returned an empty vector"
208
+ return
209
+ facts.embedding = vector
210
+ facts.embedding_status = "ok"
211
+
212
+
213
+ # ── extraction quality for pictures ──────────────────────────────────────────
214
+ def image_quality_score(facts: ImageFacts) -> Dict[str, Any]:
215
+ """``{"score": float, "reasons": [...]}`` for an image memory.
216
+
217
+ Deliberately *not* a judgement about the photograph: it scores how much of
218
+ this image the Brain can actually retrieve later. Pixels alone are worth
219
+ little to a text query, OCR text is worth the most, a caption is worth a
220
+ lot, and a vector is worth something even without either.
221
+ """
222
+ if not facts.readable:
223
+ return {"score": 0.0, "reasons": ["image_unreadable"]}
224
+ reasons: List[str] = []
225
+ score = 0.15 # we know it is an image and how big it is
226
+ if facts.ocr_text:
227
+ # 400+ characters of recognized text is a page, not a label.
228
+ score += 0.45 * min(1.0, len(facts.ocr_text) / 400.0)
229
+ reasons.append("ocr_text")
230
+ elif facts.ocr_status == "unavailable":
231
+ reasons.append("ocr_unavailable")
232
+ elif facts.ocr_status == "skipped":
233
+ reasons.append("ocr_skipped")
234
+ else:
235
+ reasons.append("no_ocr_text")
236
+ if facts.caption:
237
+ score += 0.3
238
+ reasons.append("vision_caption")
239
+ else:
240
+ reasons.append("no_vision_caption")
241
+ if facts.embedding_status == "ok":
242
+ score += 0.1
243
+ reasons.append("vision_embedding")
244
+ return {"score": round(max(0.0, min(1.0, score)), 4), "reasons": reasons}
245
+
246
+
247
+ def image_node_id(content_hash: str, workspace_id: Optional[str] = None) -> str:
248
+ """Workspace-scoped, content-addressed id — re-ingesting is idempotent."""
249
+ scoped = f"{workspace_id or 'legacy-global'}|{content_hash}"
250
+ return f"image:{_sha256_text(scoped)[:24]}"
251
+
252
+
253
+ def write_image_memory(
254
+ store: Any,
255
+ *,
256
+ path: Path,
257
+ facts: ImageFacts,
258
+ title: str,
259
+ source_type: str = MODALITY_IMAGE,
260
+ source_uri: Optional[str] = None,
261
+ owner: Optional[str] = None,
262
+ workspace_id: Optional[str] = None,
263
+ conversation_id: Optional[str] = None,
264
+ captured_at: Optional[str] = None,
265
+ modified_at: Optional[str] = None,
266
+ permissions: Optional[Dict[str, Any]] = None,
267
+ extra_metadata: Optional[Dict[str, Any]] = None,
268
+ ) -> Dict[str, Any]:
269
+ """Write one ``Image`` node (plus ``ImageText``/chunks) into the graph.
270
+
271
+ The node is the image itself rather than a ``Document`` that happens to be
272
+ a picture, because that is what the rest of the product reasons about: the
273
+ graph already declares ``Image``/``ImageText``/``CONTAINS_IMAGE``, and
274
+ ``hybrid_search`` already ranks ``Image`` as a first-class result type.
275
+
276
+ Uses the same cross-mixin write door (``_upsert_node``/``_upsert_edge``/
277
+ ``_upsert_chunk``) that every other ingest path uses — the store exposes no
278
+ public node writer, and inventing a second one for images would be a
279
+ parallel write path to keep in sync forever.
280
+ """
281
+ captured_at = captured_at or utc_now_iso()
282
+ content_hash = _sha256_file(path)
283
+ node_id = image_node_id(content_hash, workspace_id)
284
+ index_text = facts.index_text()
285
+ metadata: Dict[str, Any] = {
286
+ "filename": path.name,
287
+ "file_path": str(path),
288
+ "ext": path.suffix.lower(),
289
+ "bytes": path.stat().st_size,
290
+ "sha256": content_hash,
291
+ "content_hash": content_hash,
292
+ "source_type": source_type,
293
+ "source_uri": source_uri or str(path),
294
+ "captured_at": captured_at,
295
+ "modified_at": modified_at,
296
+ "owner": owner,
297
+ "workspace_id": workspace_id,
298
+ "permissions": permissions or {},
299
+ "conversation_id": conversation_id,
300
+ **facts.as_metadata(),
301
+ **(extra_metadata or {}),
302
+ }
303
+ # Honest summary: when nothing could be read out of the picture, say so
304
+ # rather than leaving a blank card that looks like a failed render.
305
+ summary = index_text[:SUMMARY_CHARS] or f"[{MODALITY_IMAGE}] {path.name}"
306
+ chunk_ids: List[str] = []
307
+
308
+ with store._connect() as conn:
309
+ duplicate = (
310
+ conn.execute("SELECT 1 FROM nodes WHERE id=? LIMIT 1", (node_id,)).fetchone()
311
+ is not None
312
+ )
313
+ store._upsert_node(
314
+ conn,
315
+ node_id,
316
+ "Image",
317
+ title or path.name,
318
+ summary=summary,
319
+ metadata=metadata,
320
+ raw=metadata,
321
+ owner=owner,
322
+ workspace_id=workspace_id,
323
+ )
324
+ if facts.ocr_text:
325
+ image_text_id = f"imagetext:{_sha256_text(f'{node_id}:ocr')[:24]}"
326
+ store._upsert_node(
327
+ conn,
328
+ image_text_id,
329
+ "ImageText",
330
+ f"{path.name} OCR",
331
+ summary=facts.ocr_text[:700],
332
+ metadata={
333
+ "source_node": node_id,
334
+ "chars": len(facts.ocr_text),
335
+ "workspace_id": workspace_id,
336
+ },
337
+ owner=owner,
338
+ workspace_id=workspace_id,
339
+ )
340
+ store._upsert_edge(
341
+ conn,
342
+ node_id,
343
+ image_text_id,
344
+ "포함함",
345
+ weight=0.8,
346
+ metadata={"source": "ocr", "workspace_id": workspace_id},
347
+ )
348
+ for index, piece in enumerate(_split_index_text(index_text)):
349
+ chunk_id = f"chunk:{_sha256_text(f'{node_id}:{index}:{piece}')[:24]}"
350
+ chunk_ids.append(chunk_id)
351
+ chunk_meta = {
352
+ "index": index,
353
+ "source_node": node_id,
354
+ "workspace_id": workspace_id,
355
+ "modality": MODALITY_IMAGE,
356
+ }
357
+ store._upsert_node(
358
+ conn,
359
+ chunk_id,
360
+ "Chunk",
361
+ f"{path.name} chunk {index + 1}",
362
+ summary=piece[:SUMMARY_CHARS],
363
+ metadata=chunk_meta,
364
+ owner=owner,
365
+ workspace_id=workspace_id,
366
+ )
367
+ store._upsert_chunk(
368
+ conn,
369
+ chunk_id=chunk_id,
370
+ source_node=node_id,
371
+ text=piece,
372
+ metadata=chunk_meta,
373
+ )
374
+ store._upsert_edge(conn, node_id, chunk_id, "포함함")
375
+ # Concepts come from what is *in* the picture, never from its name:
376
+ # "IMG_2381" is not a topic, and turning filenames into concept nodes
377
+ # would fill the graph with hubs that mean nothing.
378
+ concept_ids = _attach_concepts(
379
+ store,
380
+ conn,
381
+ node_id=node_id,
382
+ text=index_text,
383
+ owner=owner,
384
+ workspace_id=workspace_id,
385
+ )
386
+ source_node_id = _attach_source(
387
+ store,
388
+ conn,
389
+ node_id=node_id,
390
+ source_type=source_type,
391
+ source_uri=source_uri or str(path),
392
+ title=title or path.name,
393
+ content_hash=content_hash,
394
+ captured_at=captured_at,
395
+ owner=owner,
396
+ workspace_id=workspace_id,
397
+ )
398
+ metadata["concepts"] = concept_ids
399
+ return {
400
+ "node_id": node_id,
401
+ "type": "Image",
402
+ "title": title or path.name,
403
+ "sha256": content_hash,
404
+ "content_hash": content_hash,
405
+ "source_node_id": source_node_id,
406
+ "chunk_ids": chunk_ids,
407
+ "chunk_count": len(chunk_ids),
408
+ "duplicate": duplicate,
409
+ "captured_at": captured_at,
410
+ "metadata": metadata,
411
+ }
412
+
413
+
414
+ def _attach_concepts(
415
+ store: Any,
416
+ conn: Any,
417
+ *,
418
+ node_id: str,
419
+ text: str,
420
+ owner: Optional[str],
421
+ workspace_id: Optional[str],
422
+ ) -> List[str]:
423
+ """Pull concepts out of the caption/OCR so the picture joins the graph.
424
+
425
+ This is what "the caption contributes to the graph" means concretely: the
426
+ same extractor every text door uses, over the same text, producing the same
427
+ ``Concept``/``Feature``/… nodes and ``포함함`` edges — so a photo of a
428
+ whiteboard about Q3 planning is one hop from every note about Q3 planning,
429
+ instead of being an island that only exact search can reach.
430
+
431
+ An image with nothing readable in it yields no concepts, which is correct:
432
+ there is nothing to say about it.
433
+ """
434
+ body = str(text or "").strip()
435
+ if not body:
436
+ return []
437
+ from ..graph._kg_common import ( # local: keeps this module's import light
438
+ _classify_node_type,
439
+ _extract_concepts,
440
+ )
441
+
442
+ # The id derivation is imported rather than re-derived: two copies of a
443
+ # node-id rule diverge, and a diverged id is a duplicate concept nobody
444
+ # can see.
445
+ from ..graph.ingest import _scoped_slug_id
446
+
447
+ concept_ids: List[str] = []
448
+ for concept in _extract_concepts(body, limit=10):
449
+ node_type = _classify_node_type(concept, body)
450
+ concept_id = _scoped_slug_id(node_type.lower(), concept, workspace_id)
451
+ store._upsert_node(
452
+ conn,
453
+ concept_id,
454
+ node_type,
455
+ concept,
456
+ metadata={
457
+ "auto_extracted": True,
458
+ "source_node": node_id,
459
+ "modality": MODALITY_IMAGE,
460
+ "workspace_id": workspace_id,
461
+ },
462
+ owner=owner,
463
+ workspace_id=workspace_id,
464
+ )
465
+ store._upsert_edge(conn, node_id, concept_id, "포함함", weight=0.8)
466
+ concept_ids.append(concept_id)
467
+ return concept_ids
468
+
469
+
470
+ def _attach_source(
471
+ store: Any,
472
+ conn: Any,
473
+ *,
474
+ node_id: str,
475
+ source_type: str,
476
+ source_uri: str,
477
+ title: str,
478
+ content_hash: str,
479
+ captured_at: str,
480
+ owner: Optional[str],
481
+ workspace_id: Optional[str],
482
+ ) -> Optional[str]:
483
+ """Link the image to a ``Source`` node when the store can make one."""
484
+ attach = getattr(store, "_attach_source_node", None)
485
+ if not callable(attach):
486
+ return None
487
+ return str(
488
+ attach(
489
+ conn,
490
+ node_id,
491
+ source_type=source_type,
492
+ source_uri=source_uri,
493
+ title=title,
494
+ content_hash=content_hash,
495
+ captured_at=captured_at,
496
+ extra={"owner": owner, "workspace_id": workspace_id, "modality": MODALITY_IMAGE},
497
+ )
498
+ )