ltcai 11.0.1 → 11.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -43
- package/docs/CHANGELOG.md +61 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/FEATURE_AUDIT_v11.2.0.md +393 -0
- package/docs/LAYOUT_REBUILD_SPEC.md +9 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +1 -1
- package/docs/PERFORMANCE.md +71 -18
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/architecture.md +6 -2
- package/docs/kg-schema.md +1 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/embeddings.py +12 -37
- package/lattice_brain/gates.py +125 -0
- package/lattice_brain/graph/discovery_index.py +30 -32
- package/lattice_brain/graph/fusion.py +35 -4
- package/lattice_brain/graph/image_vectors.py +230 -0
- package/lattice_brain/graph/ingest.py +11 -5
- package/lattice_brain/graph/projection.py +66 -8
- package/lattice_brain/graph/provenance.py +27 -2
- package/lattice_brain/graph/retrieval.py +113 -2
- package/lattice_brain/graph/retrieval_docgen.py +6 -6
- package/lattice_brain/graph/schema.py +18 -0
- package/lattice_brain/graph/store.py +9 -0
- package/lattice_brain/graph/vector_index/selector.py +32 -2
- package/lattice_brain/ingestion.py +363 -10
- package/lattice_brain/multimodal.py +1258 -0
- package/lattice_brain/portability.py +169 -32
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/lattice_brain/sealed_box.py +244 -0
- package/lattice_brain/self_model.py +77 -22
- package/lattice_brain/synthesis.py +24 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/brain_intelligence.py +4 -0
- package/latticeai/api/chat.py +11 -0
- package/latticeai/api/chat_helpers.py +16 -3
- package/latticeai/api/chat_hybrid.py +32 -1
- package/latticeai/api/features.py +70 -0
- package/latticeai/api/local_files.py +102 -0
- package/latticeai/api/memory.py +128 -1
- package/latticeai/api/portability.py +39 -4
- package/latticeai/api/review_queue.py +126 -0
- package/latticeai/api/search.py +16 -2
- package/latticeai/core/agent.py +59 -2
- package/latticeai/core/agent_prompts.py +66 -0
- package/latticeai/core/config.py +4 -1
- package/latticeai/core/context_builder.py +98 -11
- package/latticeai/core/embedding_providers.py +528 -0
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/messages.py +180 -0
- package/latticeai/core/model_compat.py +73 -2
- package/latticeai/core/workspace_os.py +43 -0
- package/latticeai/core/workspace_os_constants.py +1 -1
- package/latticeai/core/workspace_reorganization.py +335 -0
- package/latticeai/models/model_providers.py +12 -4
- package/latticeai/runtime/build_phases.py +33 -2
- package/latticeai/runtime/chat_wiring.py +4 -0
- package/latticeai/runtime/feature_toggle_wiring.py +163 -0
- package/latticeai/runtime/persistence_runtime.py +41 -4
- package/latticeai/runtime/router_registration.py +11 -0
- package/latticeai/runtime/runtime_context.py +1 -0
- package/latticeai/services/app_context.py +8 -0
- package/latticeai/services/architecture_readiness.py +1 -1
- package/latticeai/services/automation_intelligence.py +22 -2
- package/latticeai/services/brain_intelligence.py +123 -7
- package/latticeai/services/change_proposals.py +50 -10
- package/latticeai/services/command_center.py +10 -4
- package/latticeai/services/feature_toggles.py +502 -0
- package/latticeai/services/folder_watch.py +122 -1
- package/latticeai/services/hybrid_chat.py +56 -5
- package/latticeai/services/interop_bridges.py +978 -0
- package/latticeai/services/memory_service.py +34 -0
- package/latticeai/services/model_capability_registry.py +434 -261
- package/latticeai/services/model_catalog.py +95 -61
- package/latticeai/services/model_recommendation.py +18 -11
- package/latticeai/services/model_runtime.py +1 -1
- package/latticeai/services/multimodal_ports.py +112 -0
- package/latticeai/services/obsidian_bridge.py +16 -25
- package/latticeai/services/product_readiness.py +1 -1
- package/latticeai/services/search_service.py +149 -2
- package/latticeai/services/self_model_service.py +171 -0
- package/latticeai/services/tool_dispatch.py +4 -0
- package/latticeai/services/voice_capture.py +27 -1
- package/latticeai/setup/auto_setup.py +27 -30
- package/latticeai/setup/wizard.py +77 -44
- package/package.json +1 -1
- package/scripts/check_current_release_docs.mjs +1 -1
- package/scripts/check_server_i18n.mjs +1 -0
- package/scripts/release_screen_claims.json +22 -0
- package/scripts/verify_hf_model_registry.py +253 -218
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +37 -37
- package/static/app/assets/{Act-D4zSxFR-.js → Act-AWf0SAKp.js} +1 -1
- package/static/app/assets/{AdminConsole-w5jBfPt2.js → AdminConsole-D0u8Tiyj.js} +1 -1
- package/static/app/assets/{Brain-C2EqQg74.js → Brain-tuhI4sOC.js} +1 -1
- package/static/app/assets/BrainHome-Ts7G_Ila.js +2 -0
- package/static/app/assets/BrainSignals-jMYgQ2Ar.js +1 -0
- package/static/app/assets/{Capture-DPqpGK8d.js → Capture-CqOSzyPr.js} +1 -1
- package/static/app/assets/{CommandPalette-CNf7h5fp.js → CommandPalette-DC0Bzh-I.js} +1 -1
- package/static/app/assets/{Library-BN0HYOfc.js → Library-CX-bbhmK.js} +1 -1
- package/static/app/assets/{LivingBrain-Dfq_wEDI.js → LivingBrain-DBwhto14.js} +1 -1
- package/static/app/assets/{ProductFlow-B-3O0rNV.js → ProductFlow-BHA2cfKI.js} +1 -1
- package/static/app/assets/{ReviewCard-gZ-tdqFM.js → ReviewCard-BUhCKRNM.js} +1 -1
- package/static/app/assets/{System-BElUcSSw.js → System-Bu2t5hn1.js} +1 -1
- package/static/app/assets/arrow-left-Dzwa5zRb.js +1 -0
- package/static/app/assets/{bot--qYHMtkP.js → bot-Cia42c2h.js} +1 -1
- package/static/app/assets/brain-DJMoqrwx.js +1 -0
- package/static/app/assets/{button-51Z3rsuv.js → button-2j2Ijzgq.js} +1 -1
- package/static/app/assets/{circle-pause-CMIiMaQl.js → circle-pause-BEFeWpVW.js} +1 -1
- package/static/app/assets/{circle-play-DZoO_cfG.js → circle-play-ujXMcHxl.js} +1 -1
- package/static/app/assets/{cpu-Bs6uc9W9.js → cpu-k4awryFq.js} +1 -1
- package/static/app/assets/{download-G-2olkWz.js → download-DFbLJ_ig.js} +1 -1
- package/static/app/assets/{folder-open-CTOspnmb.js → folder-open-7y_b6xkM.js} +1 -1
- package/static/app/assets/{hard-drive-CewHWJhn.js → hard-drive-Bidh02Kr.js} +1 -1
- package/static/app/assets/{index-D7Rr-J2Y.js → index-BpYkzcVm.js} +3 -3
- package/static/app/assets/index-DwDl9-8Y.css +2 -0
- package/static/app/assets/{input-D4w_BZWl.js → input-DSlJJxRs.js} +1 -1
- package/static/app/assets/{permissionCopy-CosBEXAZ.js → permissionCopy-Bpb83Hx9.js} +1 -1
- package/static/app/assets/{primitives-d0g9pvzS.js → primitives-BCx6TvfG.js} +1 -1
- package/static/app/assets/search-Cgy8cCFJ.js +1 -0
- package/static/app/assets/{share-2-NmD7e_oV.js → share-2-BH1M-WNi.js} +1 -1
- package/static/app/assets/{shield-alert-CcQeMuju.js → shield-alert-BlKdBXcG.js} +1 -1
- package/static/app/assets/{textarea-BPAJDc-0.js → textarea-CCWbUfFB.js} +1 -1
- package/static/app/assets/{useFocusTrap-C7YLdTBC.js → useFocusTrap-YdHQ7pJ1.js} +1 -1
- package/static/app/assets/{useQuery-DRyD9opW.js → useQuery-CXQiwbVT.js} +1 -1
- package/static/app/assets/{utils-DG1_ExrP.js → utils-zqPZJxdx.js} +2 -2
- package/static/app/assets/{workspace-CWVf3gsI.js → workspace-DXTihhfU.js} +1 -1
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/static/app/assets/BrainHome-CvXS6XiQ.js +0 -2
- package/static/app/assets/BrainSignals-DOE_KhOU.js +0 -1
- package/static/app/assets/arrow-left-CFNIMjhv.js +0 -1
- package/static/app/assets/brain-DDCLjRqO.js +0 -1
- package/static/app/assets/index-CkzokZAj.css +0 -2
- package/static/app/assets/search-BLCYt75v.js +0 -1
|
@@ -94,64 +94,83 @@ _RAW_ENGINE_MODEL_CATALOG: Dict[str, List[Dict[str, Any]]] = _build_engine_model
|
|
|
94
94
|
# are all defined; declared here so the public name exists for static readers.
|
|
95
95
|
ENGINE_MODEL_CATALOG: Dict[str, List[Dict[str, Any]]] = {}
|
|
96
96
|
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
"
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
"
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
"
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
"
|
|
126
|
-
|
|
127
|
-
"
|
|
128
|
-
"
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
"
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
"
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
"
|
|
139
|
-
|
|
97
|
+
# Per-engine repo ids for the recommended catalog. Every repo below was resolved
|
|
98
|
+
# through the HF API on 2026-08-10 and is stored in the Hub's canonical casing
|
|
99
|
+
# (`google/gemma-4-12b-it` answers as `google/gemma-4-12B-it`; `ggml-org/
|
|
100
|
+
# gemma-4-26b-a4b-it-GGUF` answers as `ggml-org/gemma-4-26B-A4B-it-GGUF`).
|
|
101
|
+
# Lookups are lowercase, so each model is keyed by both its short name and its
|
|
102
|
+
# full mlx repo id — see `_normalize_engine_entry` and `ModelResolution`.
|
|
103
|
+
#
|
|
104
|
+
# Removed in 11.2.0: `qwen3-vl-8b` and `llama-4-scout`. Their vllm/lmstudio
|
|
105
|
+
# targets pointed at `meta-llama/Llama-4-Scout-17B-16E-Instruct`, which is
|
|
106
|
+
# gated — the download could never succeed without Hub credentials.
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _gguf_routes(mlx_repo: str, gguf_repo: str, hf_repo: str) -> Dict[str, str]:
|
|
110
|
+
"""One model's engine map: MLX weights, a GGUF repo, and the upstream HF repo."""
|
|
111
|
+
return {
|
|
112
|
+
"local_mlx": mlx_repo,
|
|
113
|
+
"ollama": f"hf.co/{gguf_repo}:Q4_K_M",
|
|
114
|
+
"vllm": hf_repo,
|
|
115
|
+
"lmstudio": gguf_repo,
|
|
116
|
+
"llamacpp": gguf_repo,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_ENGINE_ROUTES: Dict[str, Dict[str, str]] = {
|
|
121
|
+
"mlx-community/LFM2.5-2.6B-4bit": _gguf_routes(
|
|
122
|
+
"mlx-community/LFM2.5-2.6B-4bit", "LiquidAI/LFM2.5-2.6B-GGUF", "LiquidAI/LFM2.5-2.6B",
|
|
123
|
+
),
|
|
124
|
+
"mlx-community/gemma-4-e2b-it-4bit": _gguf_routes(
|
|
125
|
+
"mlx-community/gemma-4-e2b-it-4bit", "ggml-org/gemma-4-E2B-it-GGUF", "google/gemma-4-E2B-it",
|
|
126
|
+
),
|
|
127
|
+
"mlx-community/gemma-4-e4b-it-4bit": _gguf_routes(
|
|
128
|
+
"mlx-community/gemma-4-e4b-it-4bit", "ggml-org/gemma-4-E4B-it-GGUF", "google/gemma-4-E4B-it",
|
|
129
|
+
),
|
|
130
|
+
"mlx-community/gemma-4-12B-it-4bit": _gguf_routes(
|
|
131
|
+
"mlx-community/gemma-4-12B-it-4bit", "ggml-org/gemma-4-12B-it-GGUF", "google/gemma-4-12B-it",
|
|
132
|
+
),
|
|
133
|
+
"mlx-community/gemma-4-26b-a4b-it-4bit": _gguf_routes(
|
|
134
|
+
"mlx-community/gemma-4-26b-a4b-it-4bit", "ggml-org/gemma-4-26B-A4B-it-GGUF",
|
|
135
|
+
"google/gemma-4-26B-A4B-it",
|
|
136
|
+
),
|
|
137
|
+
"mlx-community/gemma-4-31b-it-4bit": _gguf_routes(
|
|
138
|
+
"mlx-community/gemma-4-31b-it-4bit", "ggml-org/gemma-4-31B-it-GGUF", "google/gemma-4-31B-it",
|
|
139
|
+
),
|
|
140
|
+
"mlx-community/Qwen3.6-27B-4bit": _gguf_routes(
|
|
141
|
+
"mlx-community/Qwen3.6-27B-4bit", "ggml-org/Qwen3.6-27B-GGUF", "Qwen/Qwen3.6-27B",
|
|
142
|
+
),
|
|
143
|
+
"mlx-community/gpt-oss-20b-MXFP4-Q8": _gguf_routes(
|
|
144
|
+
"mlx-community/gpt-oss-20b-MXFP4-Q8", "ggml-org/gpt-oss-20b-GGUF", "openai/gpt-oss-20b",
|
|
145
|
+
),
|
|
146
|
+
# No community GGUF build verified for these two, so they offer the MLX
|
|
147
|
+
# weights and the upstream repo only rather than a route that would 404.
|
|
148
|
+
"mlx-community/Qwen3.5-9B-MLX-4bit": {
|
|
149
|
+
"local_mlx": "mlx-community/Qwen3.5-9B-MLX-4bit",
|
|
150
|
+
"vllm": "Qwen/Qwen3.5-9B",
|
|
140
151
|
},
|
|
141
|
-
"
|
|
142
|
-
"local_mlx": "mlx-community/Qwen3-
|
|
143
|
-
"
|
|
144
|
-
"vllm": "Qwen/Qwen3-VL-8B-Instruct",
|
|
145
|
-
"lmstudio": "Qwen/Qwen3-VL-8B-Instruct",
|
|
146
|
-
"llamacpp": "Qwen/Qwen3-VL-8B-Instruct-GGUF",
|
|
152
|
+
"mlx-community/Qwen3.6-35B-A3B-4bit": {
|
|
153
|
+
"local_mlx": "mlx-community/Qwen3.6-35B-A3B-4bit",
|
|
154
|
+
"vllm": "Qwen/Qwen3.6-35B-A3B",
|
|
147
155
|
},
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
# Names people actually type that are not the repo's own short name. Kept
|
|
159
|
+
# because a user who asks for "gemma-4-26b-it-4bit" means the A4B build — there
|
|
160
|
+
# is no other Gemma 4 26B — and resolving that to nothing would be pedantry.
|
|
161
|
+
_COMMON_MISNAMES: Dict[str, str] = {
|
|
162
|
+
"gemma-4-26b-it-4bit": "mlx-community/gemma-4-26b-a4b-it-4bit",
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
# Keyed by both the lowercase short name ("gemma-4-12b-it-4bit") and the
|
|
166
|
+
# lowercase full repo id, because callers reach this map from both directions.
|
|
167
|
+
MODEL_ENGINE_ALIASES: Dict[str, Dict[str, str]] = {
|
|
168
|
+
**{
|
|
169
|
+
key: routes
|
|
170
|
+
for repo, routes in _ENGINE_ROUTES.items()
|
|
171
|
+
for key in (repo.split("/")[-1].lower(), repo.lower())
|
|
154
172
|
},
|
|
173
|
+
**{alias: _ENGINE_ROUTES[repo] for alias, repo in _COMMON_MISNAMES.items()},
|
|
155
174
|
}
|
|
156
175
|
|
|
157
176
|
# Also expose registry helpers directly from here for consumers who want the rich objects
|
|
@@ -181,7 +200,14 @@ def _model_family_version(model: Dict[str, Any]) -> Optional[tuple[str, tuple[in
|
|
|
181
200
|
# Every pattern captures at least one decimal digit and no other
|
|
182
201
|
# character but ``.``, so the parsed tuple is never empty — the
|
|
183
202
|
# old ``if version:`` guard could not fire and is gone.
|
|
184
|
-
|
|
203
|
+
#
|
|
204
|
+
# Compared at *major* granularity only (``[:1]``). The filter exists
|
|
205
|
+
# to hide superseded generations — Qwen 2.5 behind Qwen 3, Gemma 3
|
|
206
|
+
# behind Gemma 4 — not to pick a winner inside one generation.
|
|
207
|
+
# Comparing minors made Qwen3.5 9B (the mid VLM) vanish the moment
|
|
208
|
+
# Qwen3.6 27B joined the catalog, even though they fill different
|
|
209
|
+
# RAM tiers and ship side by side.
|
|
210
|
+
return family, _version_tuple(match.group(1))[:1]
|
|
185
211
|
return None
|
|
186
212
|
|
|
187
213
|
|
|
@@ -202,16 +228,24 @@ def filter_lower_family_versions(models: List[Dict[str, Any]]) -> List[Dict[str,
|
|
|
202
228
|
]
|
|
203
229
|
|
|
204
230
|
|
|
205
|
-
# ──
|
|
206
|
-
#
|
|
207
|
-
#
|
|
208
|
-
#
|
|
231
|
+
# ── User-facing catalog assembly ──────────────────────────────────────────────
|
|
232
|
+
# The capability registry already keeps superseded generations out of the
|
|
233
|
+
# catalog by lifecycle (they live in its LEGACY list, recognised but never
|
|
234
|
+
# offered). This blocklist is the second lock: a regression guard so a retired
|
|
235
|
+
# generation cannot creep back into the model picker through a hand-edited
|
|
236
|
+
# entry. Anything whose id contains one of these fragments is dropped from
|
|
237
|
+
# ENGINE_MODEL_CATALOG.
|
|
238
|
+
#
|
|
239
|
+
# 11.2.0 removed ``gpt-oss`` from this list — GPT-OSS 20B is now a recommended
|
|
240
|
+
# entry, not a retired one — and added the generations retired this release.
|
|
209
241
|
_BLOCKED_CATALOG_FRAGMENTS = (
|
|
210
242
|
"gemma-3", "gemma3", "gemma-2", "gemma2",
|
|
211
243
|
"qwen2.5", "qwen-2.5", "qwen2-5",
|
|
244
|
+
"qwen3-vl", "qwen3vl",
|
|
212
245
|
"llama-3", "llama3.2", "llama-3.2",
|
|
246
|
+
"llama-4", "llama4",
|
|
213
247
|
"pixtral", "mistral",
|
|
214
|
-
"smollm", "
|
|
248
|
+
"smollm", "phi-", "moondream",
|
|
215
249
|
)
|
|
216
250
|
|
|
217
251
|
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
Given a detected system profile (from :func:`auto_setup.probe`) this module
|
|
4
4
|
classifies every model in :data:`model_catalog.ENGINE_MODEL_CATALOG` into one of
|
|
5
5
|
three states — **recommended**, **compatible**, or **not_recommended** — and
|
|
6
|
-
groups the result by current
|
|
7
|
-
|
|
6
|
+
groups the result by current model family (Gemma 4, Qwen3.6, Qwen3.5, GPT-OSS,
|
|
7
|
+
LFM2.5).
|
|
8
8
|
|
|
9
9
|
It is intentionally pure and dependency-light: the only input is a plain dict
|
|
10
10
|
describing the machine, so it is fully unit-testable without touching real
|
|
@@ -30,18 +30,25 @@ NOT_RECOMMENDED = "not_recommended"
|
|
|
30
30
|
# Apple-Silicon only. Used to decide platform availability before sizing.
|
|
31
31
|
_APPLE_ONLY_ENGINES = {"local_mlx"}
|
|
32
32
|
|
|
33
|
-
# Family display order for the grouped view (
|
|
33
|
+
# Family display order for the grouped view (current generations, largest
|
|
34
|
+
# lineup first). Superseded families are not listed because the capability
|
|
35
|
+
# registry never lets them reach the catalog — they are recognised for loading
|
|
36
|
+
# only. A family missing from this list still renders, it just sorts last.
|
|
34
37
|
_FAMILY_ORDER = [
|
|
35
38
|
"Gemma 4",
|
|
36
|
-
"Qwen3
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
-
"Phi Vision",
|
|
41
|
-
"Moondream",
|
|
42
|
-
"Pixtral",
|
|
39
|
+
"Qwen3.6",
|
|
40
|
+
"Qwen3.5",
|
|
41
|
+
"GPT-OSS",
|
|
42
|
+
"LFM2.5",
|
|
43
43
|
]
|
|
44
44
|
|
|
45
|
+
# Modalities the recommender will surface. Text-only models earn their place —
|
|
46
|
+
# LFM2.5 is the only thing that runs comfortably on 8GB, and GPT-OSS 20B is the
|
|
47
|
+
# most-downloaded entry in the catalog — so filtering to `multimodal` would have
|
|
48
|
+
# hidden two of the tiers. Each row still reports its own `modality`, so a UI
|
|
49
|
+
# that wants to badge "reads pictures" can, honestly.
|
|
50
|
+
_RECOMMENDABLE_MODALITIES = {"multimodal", "text"}
|
|
51
|
+
|
|
45
52
|
_SIZE_RE = re.compile(r"([\d.]+)\s*(TB|GB|MB)", re.IGNORECASE)
|
|
46
53
|
_UNIT_GB = {"TB": 1024.0, "GB": 1.0, "MB": 1.0 / 1024.0}
|
|
47
54
|
|
|
@@ -165,7 +172,7 @@ def recommend_catalog(profile: Dict[str, Any], *, engine: str = "local_mlx") ->
|
|
|
165
172
|
"""
|
|
166
173
|
models = [
|
|
167
174
|
model for model in ENGINE_MODEL_CATALOG.get(engine, [])
|
|
168
|
-
if str(model.get("modality") or "").lower()
|
|
175
|
+
if str(model.get("modality") or "").lower() in _RECOMMENDABLE_MODALITIES
|
|
169
176
|
]
|
|
170
177
|
engine_available = _engine_available(engine, profile)
|
|
171
178
|
ram_gb = _ram_gb(profile)
|
|
@@ -138,7 +138,7 @@ class ModelRuntimeState:
|
|
|
138
138
|
ALLOW_PLAINTEXT_API_KEYS: bool = False
|
|
139
139
|
CORS_ALLOW_NETWORK: bool = False
|
|
140
140
|
PUBLIC_MODEL: str = "openai:gpt-4o-mini"
|
|
141
|
-
LOCAL_MODEL: str = "mlx-community/gemma-4-
|
|
141
|
+
LOCAL_MODEL: str = "mlx-community/gemma-4-12B-it-4bit"
|
|
142
142
|
IS_PUBLIC_MODE: bool = False
|
|
143
143
|
keyring: Any = None
|
|
144
144
|
get_current_user: Callable[[Any], Optional[str]] = _missing_current_user
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Where the app decides what it can actually see and hear (v11.1.0).
|
|
2
|
+
|
|
3
|
+
Brain Core owns the *shape* of a multi-modal memory; it deliberately owns none
|
|
4
|
+
of the models. This module is the one place that turns configuration into the
|
|
5
|
+
plain callables :class:`lattice_brain.multimodal.MultimodalPorts` accepts, so
|
|
6
|
+
the ingestion pipeline never learns that ``latticeai`` exists.
|
|
7
|
+
|
|
8
|
+
Everything here is off by default and costs nothing when off: with no vision
|
|
9
|
+
provider configured, :func:`build_multimodal_ports` does no model loading, no
|
|
10
|
+
imports of optional packages, and no network calls — it returns a bundle whose
|
|
11
|
+
every capability is honestly ``None``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
18
|
+
|
|
19
|
+
from lattice_brain.ingestion import ALLOW_MULTIMODAL_ENV
|
|
20
|
+
from lattice_brain.multimodal import MODALITY_IMAGE, MultimodalPorts
|
|
21
|
+
from latticeai.core.embedding_providers import (
|
|
22
|
+
VISION_CAPTION_TARGET_ENV,
|
|
23
|
+
resolve_vision_captioner,
|
|
24
|
+
resolve_vision_embedder,
|
|
25
|
+
vision_caption_port,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
#: ``mlx`` | ``custom`` | "" (off). Never defaulted to something that loads.
|
|
29
|
+
VISION_PROVIDER_ENV = "LATTICEAI_VISION_PROVIDER"
|
|
30
|
+
VISION_MODEL_ENV = "LATTICEAI_VISION_MODEL"
|
|
31
|
+
#: ``image`` (own index + late fusion) or ``shared`` (same space as text).
|
|
32
|
+
VISION_SPACE_ENV = "LATTICEAI_VISION_SPACE"
|
|
33
|
+
VISION_CAPTION_PROVIDER_ENV = "LATTICEAI_VISION_CAPTION_PROVIDER"
|
|
34
|
+
VISION_CAPTION_MODEL_ENV = "LATTICEAI_VISION_CAPTION_MODEL"
|
|
35
|
+
|
|
36
|
+
_TRUE = {"1", "true", "yes", "on"}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def multimodal_enabled() -> bool:
|
|
40
|
+
"""Whether pictures and recordings may be ingested at all (default no)."""
|
|
41
|
+
return os.getenv(ALLOW_MULTIMODAL_ENV, "0").strip().lower() in _TRUE
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_multimodal_ports(
|
|
45
|
+
*, transcriber: Optional[Callable[[str], str]] = None
|
|
46
|
+
) -> MultimodalPorts:
|
|
47
|
+
"""Resolve the vision/audio capabilities this install really has.
|
|
48
|
+
|
|
49
|
+
``transcriber`` comes from :class:`~latticeai.services.voice_capture.
|
|
50
|
+
VoiceCaptureService` so a voice memo and a scanned ``.m4a`` are transcribed
|
|
51
|
+
by the same thing — or, far more often, by the same nothing.
|
|
52
|
+
"""
|
|
53
|
+
resolved = resolve_vision_embedder(
|
|
54
|
+
os.getenv(VISION_PROVIDER_ENV, ""),
|
|
55
|
+
model=os.getenv(VISION_MODEL_ENV, ""),
|
|
56
|
+
space=os.getenv(VISION_SPACE_ENV, MODALITY_IMAGE),
|
|
57
|
+
)
|
|
58
|
+
captioner = resolve_vision_captioner(
|
|
59
|
+
os.getenv(VISION_CAPTION_PROVIDER_ENV, ""),
|
|
60
|
+
model=os.getenv(VISION_CAPTION_MODEL_ENV, ""),
|
|
61
|
+
target=os.getenv(VISION_CAPTION_TARGET_ENV, ""),
|
|
62
|
+
)
|
|
63
|
+
provider = resolved.provider
|
|
64
|
+
return MultimodalPorts(
|
|
65
|
+
captioner=vision_caption_port(captioner),
|
|
66
|
+
vision_embedder=resolved.as_port(),
|
|
67
|
+
transcriber=transcriber,
|
|
68
|
+
text_to_image_embedder=text_to_image_port(provider),
|
|
69
|
+
vision_model_id=provider.model_id if provider is not None else "",
|
|
70
|
+
vision_space=resolved.space,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def text_to_image_port(
|
|
75
|
+
provider: Any,
|
|
76
|
+
) -> Optional[Callable[[str], List[float]]]:
|
|
77
|
+
"""A *query-text* → image-space vector port, only from a shared-space model.
|
|
78
|
+
|
|
79
|
+
This is what lets a typed question reach a picture through the image index
|
|
80
|
+
instead of only through its OCR text. It exists as its own port precisely
|
|
81
|
+
because most vision models cannot do it: a CLIP image vector and a BGE text
|
|
82
|
+
vector are not comparable, and `VisionEmbeddingProvider.embed_batch` refuses
|
|
83
|
+
outright unless the model declares a shared space. An image-space model
|
|
84
|
+
therefore yields ``None`` here — the honest answer, and the one the search
|
|
85
|
+
surface reports rather than silently ranking on a meaningless number.
|
|
86
|
+
"""
|
|
87
|
+
if provider is None or not getattr(provider, "shares_text_space", False):
|
|
88
|
+
return None
|
|
89
|
+
|
|
90
|
+
def _embed(query: str) -> List[float]:
|
|
91
|
+
vectors = provider.embed_batch([str(query or "")])
|
|
92
|
+
return [float(value) for value in (vectors[0] if vectors else [])]
|
|
93
|
+
|
|
94
|
+
return _embed
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def describe_multimodal(ports: MultimodalPorts) -> Dict[str, Any]:
|
|
98
|
+
"""Operator-facing status: enabled, plus what is actually wired."""
|
|
99
|
+
return {"enabled": multimodal_enabled(), **ports.describe()}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
__all__ = [
|
|
103
|
+
"VISION_CAPTION_MODEL_ENV",
|
|
104
|
+
"VISION_CAPTION_PROVIDER_ENV",
|
|
105
|
+
"VISION_MODEL_ENV",
|
|
106
|
+
"VISION_PROVIDER_ENV",
|
|
107
|
+
"VISION_SPACE_ENV",
|
|
108
|
+
"build_multimodal_ports",
|
|
109
|
+
"describe_multimodal",
|
|
110
|
+
"multimodal_enabled",
|
|
111
|
+
"text_to_image_port",
|
|
112
|
+
]
|
|
@@ -35,7 +35,6 @@ be a lie in the summary.
|
|
|
35
35
|
|
|
36
36
|
from __future__ import annotations
|
|
37
37
|
|
|
38
|
-
import json
|
|
39
38
|
import os
|
|
40
39
|
import re
|
|
41
40
|
from dataclasses import dataclass, field
|
|
@@ -48,6 +47,7 @@ from urllib.parse import unquote
|
|
|
48
47
|
# store-extracted topic the same node instead of two nodes with one label.
|
|
49
48
|
from lattice_brain.graph.ingest import _scoped_slug_id
|
|
50
49
|
from lattice_brain.ingestion import IngestionItem
|
|
50
|
+
from latticeai.services.interop_bridges import edge_row, topic_row
|
|
51
51
|
|
|
52
52
|
SOURCE_TYPE = "obsidian"
|
|
53
53
|
NOTE_EXTENSIONS = frozenset({".md", ".markdown"})
|
|
@@ -567,15 +567,9 @@ class ObsidianVaultBridge:
|
|
|
567
567
|
"index": outcome.get("index"),
|
|
568
568
|
}
|
|
569
569
|
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
"from_node": from_id,
|
|
574
|
-
"to_node": to_id,
|
|
575
|
-
"type": relation,
|
|
576
|
-
"weight": 1.0,
|
|
577
|
-
"metadata_json": json.dumps(metadata, ensure_ascii=False),
|
|
578
|
-
}
|
|
570
|
+
# Row shapes are shared with every other bridge (v11.2.0): two copies of
|
|
571
|
+
# "what an edge row looks like" is two places to forget an encoding.
|
|
572
|
+
_edge_row = staticmethod(edge_row)
|
|
579
573
|
|
|
580
574
|
@staticmethod
|
|
581
575
|
def _topic_row(
|
|
@@ -586,21 +580,18 @@ class ObsidianVaultBridge:
|
|
|
586
580
|
owner: Optional[str],
|
|
587
581
|
workspace_id: Optional[str],
|
|
588
582
|
) -> Dict[str, Any]:
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
"
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
"metadata_json": json.dumps(metadata, ensure_ascii=False),
|
|
602
|
-
"raw_json": "{}",
|
|
603
|
-
}
|
|
583
|
+
return topic_row(
|
|
584
|
+
topic_id,
|
|
585
|
+
tag,
|
|
586
|
+
summary=f"Obsidian tag #{tag}",
|
|
587
|
+
metadata={
|
|
588
|
+
"topic": tag,
|
|
589
|
+
"source": SOURCE_TYPE,
|
|
590
|
+
"vault": vault,
|
|
591
|
+
"owner": owner,
|
|
592
|
+
"workspace_id": workspace_id,
|
|
593
|
+
},
|
|
594
|
+
)
|
|
604
595
|
|
|
605
596
|
|
|
606
597
|
__all__ = [
|
|
@@ -8,9 +8,11 @@ from __future__ import annotations
|
|
|
8
8
|
|
|
9
9
|
from dataclasses import dataclass
|
|
10
10
|
from datetime import datetime
|
|
11
|
-
from typing import Any, Dict, List, Mapping, Optional
|
|
11
|
+
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
|
12
12
|
|
|
13
|
+
from lattice_brain.gates import FeatureGate
|
|
13
14
|
from lattice_brain.graph._kg_fsutil import _parse_iso, _recency_score
|
|
15
|
+
from lattice_brain.graph.image_vectors import DEFAULT_IMAGE_FUSION_WEIGHT
|
|
14
16
|
from lattice_brain.graph.retrieval_policy import resolve_policy
|
|
15
17
|
|
|
16
18
|
DEFAULT_HYBRID_WEIGHTS = {
|
|
@@ -19,6 +21,33 @@ DEFAULT_HYBRID_WEIGHTS = {
|
|
|
19
21
|
"graph": 0.25,
|
|
20
22
|
}
|
|
21
23
|
|
|
24
|
+
# ── text query → image space (v11.2.0) ───────────────────────────────────────
|
|
25
|
+
# 11.1.0 shipped image late fusion as API-only, because the two vector spaces
|
|
26
|
+
# are not comparable and nothing produced a query vector for the image index.
|
|
27
|
+
# A *shared-space* vision model can produce one, and this is the switch that
|
|
28
|
+
# lets the search API do it automatically. Off by default and resolved when
|
|
29
|
+
# asked, so a settings toggle can move it without a restart; with it off the
|
|
30
|
+
# hybrid response is byte-identical to 11.1.0's.
|
|
31
|
+
TEXT_IMAGE_FUSION_ENV = "LATTICEAI_TEXT_IMAGE_FUSION"
|
|
32
|
+
IMAGE_QUERY_FUSION_GATE = FeatureGate(
|
|
33
|
+
TEXT_IMAGE_FUSION_ENV,
|
|
34
|
+
default=False,
|
|
35
|
+
name="text_image_fusion",
|
|
36
|
+
detail=(
|
|
37
|
+
"Typed questions can also be scored against the image index when a "
|
|
38
|
+
"shared-space vision model is configured."
|
|
39
|
+
),
|
|
40
|
+
)
|
|
41
|
+
IMAGE_FUSION_UNAVAILABLE = (
|
|
42
|
+
"no shared-space vision model is configured, so a typed question cannot be "
|
|
43
|
+
"scored against image vectors; pictures are still found through their OCR "
|
|
44
|
+
"text and captions"
|
|
45
|
+
)
|
|
46
|
+
IMAGE_FUSION_DISABLED = (
|
|
47
|
+
"automatic image fusion is off for this install "
|
|
48
|
+
f"({TEXT_IMAGE_FUSION_ENV}); the caller may still supply an image vector"
|
|
49
|
+
)
|
|
50
|
+
|
|
22
51
|
|
|
23
52
|
def _clean(text: Any, limit: int = 1000) -> str:
|
|
24
53
|
return " ".join(str(text or "").split())[:limit]
|
|
@@ -31,6 +60,115 @@ def _result_key(result: Mapping[str, Any]) -> str:
|
|
|
31
60
|
@dataclass
|
|
32
61
|
class SearchService:
|
|
33
62
|
graph_store: Any
|
|
63
|
+
#: ``(query_text) -> image-space vector``. Defaults to the port the store
|
|
64
|
+
#: already carries (``multimodal_ports.text_to_image_embedder``), so no new
|
|
65
|
+
#: wiring is needed; the field exists so a test — or an install with its
|
|
66
|
+
#: own encoder — can supply one directly.
|
|
67
|
+
image_query_embedder: Any = None
|
|
68
|
+
|
|
69
|
+
# ── image-space query channel (v11.2.0) ──────────────────────────────
|
|
70
|
+
def image_query_status(self) -> Dict[str, Any]:
|
|
71
|
+
"""Whether a typed question can reach the image index here, and why not."""
|
|
72
|
+
port, detail = self._image_query_port()
|
|
73
|
+
return {
|
|
74
|
+
"available": port is not None,
|
|
75
|
+
"gate": IMAGE_QUERY_FUSION_GATE.describe(),
|
|
76
|
+
"detail": detail,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
def _image_query_port(self) -> Tuple[Any, Optional[str]]:
|
|
80
|
+
"""The text→image port and, when there is none, the honest reason."""
|
|
81
|
+
if self.image_query_embedder is not None:
|
|
82
|
+
return self.image_query_embedder, None
|
|
83
|
+
ports = getattr(self.graph_store, "multimodal_ports", None)
|
|
84
|
+
port = getattr(ports, "text_to_image_embedder", None)
|
|
85
|
+
return (port, None) if port is not None else (None, IMAGE_FUSION_UNAVAILABLE)
|
|
86
|
+
|
|
87
|
+
def _image_query_vector(self, query: str) -> Tuple[Optional[List[float]], Optional[str]]:
|
|
88
|
+
port, detail = self._image_query_port()
|
|
89
|
+
if port is None:
|
|
90
|
+
return None, detail
|
|
91
|
+
try:
|
|
92
|
+
vector = [float(value) for value in (port(query) or [])]
|
|
93
|
+
except Exception as exc: # noqa: BLE001 — a broken encoder never fails a search
|
|
94
|
+
return None, f"the vision model could not embed the query: {exc}"
|
|
95
|
+
if not vector:
|
|
96
|
+
return None, "the vision model returned an empty query vector"
|
|
97
|
+
return vector, None
|
|
98
|
+
|
|
99
|
+
def _image_channel(
|
|
100
|
+
self, query: str, matches: List[Dict[str, Any]], *, requested: bool, limit: int
|
|
101
|
+
) -> Optional[Dict[str, Any]]:
|
|
102
|
+
"""Late-fuse the image index into an already-ranked match list.
|
|
103
|
+
|
|
104
|
+
Returns ``None`` when the caller did not ask, which is what keeps the
|
|
105
|
+
response byte-identical for everyone who did not opt in. When it *was*
|
|
106
|
+
asked for and could not run, it returns the reason rather than silently
|
|
107
|
+
producing the same answer as before.
|
|
108
|
+
"""
|
|
109
|
+
if not requested:
|
|
110
|
+
return None
|
|
111
|
+
report: Dict[str, Any] = {
|
|
112
|
+
"requested": True,
|
|
113
|
+
"applied": False,
|
|
114
|
+
"weight": DEFAULT_IMAGE_FUSION_WEIGHT,
|
|
115
|
+
"candidates": 0,
|
|
116
|
+
"fused": 0,
|
|
117
|
+
"detail": None,
|
|
118
|
+
}
|
|
119
|
+
if not IMAGE_QUERY_FUSION_GATE.enabled():
|
|
120
|
+
report["detail"] = IMAGE_FUSION_DISABLED
|
|
121
|
+
return {"image_fusion": report}
|
|
122
|
+
vector, detail = self._image_query_vector(query)
|
|
123
|
+
if vector is None:
|
|
124
|
+
report["detail"] = detail
|
|
125
|
+
return {"image_fusion": report}
|
|
126
|
+
from lattice_brain.graph.image_vectors import image_similarity_search
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
found = image_similarity_search(
|
|
130
|
+
self.graph_store, vector, top_k=max(1, int(limit) * 2)
|
|
131
|
+
)
|
|
132
|
+
except Exception as exc: # noqa: BLE001 — the text answer must survive
|
|
133
|
+
report["detail"] = f"image index unavailable: {exc}"
|
|
134
|
+
return {"image_fusion": report}
|
|
135
|
+
report["candidates"] = int(found.get("candidates") or 0)
|
|
136
|
+
scores = {
|
|
137
|
+
str(match["node_id"]): float(match["score"])
|
|
138
|
+
for match in found.get("matches") or []
|
|
139
|
+
}
|
|
140
|
+
report["fused"] = self._blend_image_scores(matches, scores)
|
|
141
|
+
report["applied"] = True
|
|
142
|
+
report["detail"] = found.get("detail")
|
|
143
|
+
return {"image_fusion": report}
|
|
144
|
+
|
|
145
|
+
@staticmethod
|
|
146
|
+
def _blend_image_scores(
|
|
147
|
+
matches: List[Dict[str, Any]],
|
|
148
|
+
scores: Dict[str, float],
|
|
149
|
+
*,
|
|
150
|
+
weight: float = DEFAULT_IMAGE_FUSION_WEIGHT,
|
|
151
|
+
) -> int:
|
|
152
|
+
"""The graph layer's blend, over this layer's ``source_scores`` shape.
|
|
153
|
+
|
|
154
|
+
Late fusion by construction: the two rankings were produced
|
|
155
|
+
independently and only the final numbers meet, so neither space is ever
|
|
156
|
+
asked to interpret the other's.
|
|
157
|
+
"""
|
|
158
|
+
touched = 0
|
|
159
|
+
for match in matches:
|
|
160
|
+
raw = scores.get(_result_key(match))
|
|
161
|
+
if raw is None:
|
|
162
|
+
continue
|
|
163
|
+
touched += 1
|
|
164
|
+
match.setdefault("source_scores", {})["image"] = round(raw, 6)
|
|
165
|
+
blended = (1.0 - weight) * float(match.get("score") or 0.0) + weight * raw
|
|
166
|
+
match["score"] = round(blended, 6)
|
|
167
|
+
if touched:
|
|
168
|
+
matches.sort(key=lambda item: float(item.get("score") or 0.0), reverse=True)
|
|
169
|
+
for rank, match in enumerate(matches, start=1):
|
|
170
|
+
match["rank"] = rank
|
|
171
|
+
return touched
|
|
34
172
|
|
|
35
173
|
def _require_graph(self) -> Any:
|
|
36
174
|
if self.graph_store is None:
|
|
@@ -410,6 +548,7 @@ class SearchService:
|
|
|
410
548
|
weights: Optional[Mapping[str, float]] = None,
|
|
411
549
|
allowed_workspaces=None,
|
|
412
550
|
include_legacy_global: bool = False,
|
|
551
|
+
image_fusion: bool = False,
|
|
413
552
|
) -> Dict[str, Any]:
|
|
414
553
|
# Single retrieval policy (review Wave 0.2): when the caller does not
|
|
415
554
|
# pin explicit weights, resolve the query class + per-class channel
|
|
@@ -513,7 +652,10 @@ class SearchService:
|
|
|
513
652
|
}
|
|
514
653
|
if query_class is not None:
|
|
515
654
|
match["fusion"]["query_class"] = query_class
|
|
516
|
-
|
|
655
|
+
multimodal = self._image_channel(
|
|
656
|
+
query, matches, requested=image_fusion, limit=limit,
|
|
657
|
+
)
|
|
658
|
+
report: Dict[str, Any] = {
|
|
517
659
|
"query": query,
|
|
518
660
|
"mode": "hybrid",
|
|
519
661
|
"query_class": query_class,
|
|
@@ -529,6 +671,11 @@ class SearchService:
|
|
|
529
671
|
},
|
|
530
672
|
"matches": matches,
|
|
531
673
|
}
|
|
674
|
+
# Additive key only when the caller asked, so an untouched response is
|
|
675
|
+
# the one 11.1.0 returned.
|
|
676
|
+
if multimodal is not None:
|
|
677
|
+
report["multimodal"] = multimodal
|
|
678
|
+
return report
|
|
532
679
|
|
|
533
680
|
def graph(
|
|
534
681
|
self,
|