ltcai 11.7.0 β†’ 12.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/README.md +100 -76
  2. package/docs/BENCHMARKS.md +9 -2
  3. package/docs/CHANGELOG.md +249 -0
  4. package/docs/CI_AND_RELEASE_GATES.md +126 -41
  5. package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
  6. package/docs/DEVELOPMENT.md +271 -103
  7. package/docs/ENTERPRISE.md +1 -1
  8. package/docs/LEGACY_COMPATIBILITY.md +10 -6
  9. package/docs/MULTI_AGENT_RUNTIME.md +4 -4
  10. package/docs/ONBOARDING.md +16 -4
  11. package/docs/OPERATIONS.md +14 -1
  12. package/docs/PERMISSION_MODE.md +14 -9
  13. package/docs/REALTIME_COLLABORATION.md +1 -1
  14. package/docs/ROADMAP.md +113 -0
  15. package/docs/TRUST_MODEL.md +28 -7
  16. package/docs/USABILITY_AUDIT.md +5 -0
  17. package/docs/WHY_LATTICE.md +13 -5
  18. package/docs/WORKFLOW_DESIGNER.md +2 -2
  19. package/docs/kg-schema.md +57 -7
  20. package/docs/mcp-tools.md +93 -82
  21. package/docs/security-model.md +6 -3
  22. package/lattice_brain/__init__.py +1 -1
  23. package/lattice_brain/graph/_kg_common/__init__.py +1 -54
  24. package/lattice_brain/graph/_kg_common/extraction.py +459 -105
  25. package/lattice_brain/graph/_kg_common/normalize.py +305 -0
  26. package/lattice_brain/graph/_kg_common/patterns.py +275 -0
  27. package/lattice_brain/graph/_kg_common/relations.py +12 -3
  28. package/lattice_brain/graph/_kg_common/sections.py +107 -0
  29. package/lattice_brain/graph/_kg_common/text.py +14 -450
  30. package/lattice_brain/graph/_kg_constants.py +7 -0
  31. package/lattice_brain/ingestion/__init__.py +6 -3
  32. package/lattice_brain/multimodal/__init__.py +9 -3
  33. package/latticeai/__init__.py +1 -1
  34. package/latticeai/api/agent_worker_seam.py +44 -1
  35. package/latticeai/api/models.py +18 -110
  36. package/latticeai/api/search.py +7 -30
  37. package/latticeai/api/worker_compute.py +127 -106
  38. package/latticeai/api/worker_seams.py +17 -2
  39. package/latticeai/core/embedding_providers/__init__.py +16 -0
  40. package/latticeai/core/embedding_providers/autodetect.py +302 -0
  41. package/latticeai/core/embedding_providers/base.py +25 -0
  42. package/latticeai/core/embedding_providers/profiles.py +44 -0
  43. package/latticeai/core/embedding_providers/text.py +74 -8
  44. package/latticeai/core/http_origin.py +3 -3
  45. package/latticeai/core/messages.py +0 -5
  46. package/latticeai/core/policy.py +1 -6
  47. package/latticeai/core/quiet.py +1 -20
  48. package/latticeai/core/security.py +29 -83
  49. package/latticeai/core/sessions.py +95 -4
  50. package/latticeai/core/users.py +0 -38
  51. package/latticeai/core/vector_index/__init__.py +61 -0
  52. package/latticeai/core/vector_index/hnsw.py +383 -0
  53. package/latticeai/core/vector_index/sidecar.py +329 -0
  54. package/latticeai/models/router/catalog.py +2 -2
  55. package/latticeai/models/router/generation.py +176 -30
  56. package/latticeai/models/router/loading.py +150 -9
  57. package/latticeai/runtime/access_runtime.py +7 -4
  58. package/latticeai/runtime/brain_runtime.py +43 -9
  59. package/latticeai/runtime/build_phases/features.py +8 -31
  60. package/latticeai/runtime/build_phases/foundation.py +7 -16
  61. package/latticeai/runtime/build_phases/web.py +3 -3
  62. package/latticeai/runtime/build_phases/worker_profile.py +29 -27
  63. package/latticeai/runtime/runtime_context.py +0 -2
  64. package/latticeai/services/architecture_readiness.py +18 -19
  65. package/latticeai/services/process_audit.py +1 -22
  66. package/latticeai/services/product_readiness.py +39 -12
  67. package/latticeai/services/search_service.py +7 -0
  68. package/latticeai/services/voice_capture.py +8 -28
  69. package/latticeai/tools/__init__.py +12 -47
  70. package/latticeai/tools/commands.py +9 -15
  71. package/latticeai/tools/documents.py +12 -0
  72. package/latticeai/tools/knowledge.py +0 -6
  73. package/latticeai/tools/markup.py +152 -0
  74. package/package.json +4 -5
  75. package/requirements.txt +0 -1
  76. package/scripts/check_current_release_docs.mjs +1 -1
  77. package/scripts/check_openapi_drift.mjs +3 -2
  78. package/scripts/check_server_i18n.mjs +5 -4
  79. package/scripts/compose_openapi.py +4 -1
  80. package/scripts/export_openapi.py +5 -4
  81. package/scripts/gen_worker_allowlist_fixture.py +2 -2
  82. package/scripts/openapi_route_families.json +19 -74
  83. package/scripts/publish_release.mjs +157 -0
  84. package/scripts/release_screen_claims.json +144 -28
  85. package/src-tauri/Cargo.lock +45 -10
  86. package/src-tauri/Cargo.toml +1 -1
  87. package/src-tauri/tauri.conf.json +1 -1
  88. package/static/app/asset-manifest.json +47 -41
  89. package/static/app/assets/Act-Cf1L2709.js +2 -0
  90. package/static/app/assets/AdminConsole-DPAbLTYV.js +1 -0
  91. package/static/app/assets/Brain-DqamGrj-.js +2 -0
  92. package/static/app/assets/BrainHome-MHe2_RYs.js +2 -0
  93. package/static/app/assets/BrainSignals-CQPPfyyH.js +1 -0
  94. package/static/app/assets/Capture-DGdIH_Zc.js +1 -0
  95. package/static/app/assets/Chronicle-C-UlCJoJ.js +1 -0
  96. package/static/app/assets/CommandPalette-WNT4EqUX.js +1 -0
  97. package/static/app/assets/DigitalBrainExplorer-CEBH5Cwc.js +321 -0
  98. package/static/app/assets/Library-C6xd1dlf.js +1 -0
  99. package/static/app/assets/LivingBrain-BEk-0ohw.js +1 -0
  100. package/static/app/assets/ProductFlow-CZLm5iXh.js +1 -0
  101. package/static/app/assets/QueryClientProvider-B3OjqSyJ.js +1 -0
  102. package/static/app/assets/{ReviewCard-HXRle3qq.js β†’ ReviewCard-CEHG6evf.js} +2 -2
  103. package/static/app/assets/RunsListPanel-CLtEJSRW.js +1 -0
  104. package/static/app/assets/System-CAxwBUXw.js +1 -0
  105. package/static/app/assets/WorkflowGraph-Dj10RuGE.js +1 -0
  106. package/static/app/assets/WorkflowsPanel-Kyeh_LIT.js +2 -0
  107. package/static/app/assets/actHelpers-CtSmK9Dw.js +1 -0
  108. package/static/app/assets/arrow-left-CRl5EO4D.js +1 -0
  109. package/static/app/assets/{bot-Cn8bWRuq.js β†’ bot-DhUGRel2.js} +1 -1
  110. package/static/app/assets/brain-CLkhHsHF.js +1 -0
  111. package/static/app/assets/button-CmaEqG1T.js +1 -0
  112. package/static/app/assets/circle-check-CFgejkOS.js +1 -0
  113. package/static/app/assets/{circle-pause-CmzC_apg.js β†’ circle-pause-l96izbxj.js} +1 -1
  114. package/static/app/assets/{circle-play-D8mW2aQ7.js β†’ circle-play-CrZa25_q.js} +1 -1
  115. package/static/app/assets/{cpu-DZcdd0PZ.js β†’ cpu-BaXudqwl.js} +1 -1
  116. package/static/app/assets/{download-bv1KEPGQ.js β†’ download-hCVFPiyc.js} +1 -1
  117. package/static/app/assets/{folder-open-d-Pip5gr.js β†’ folder-open-CHL82Yp7.js} +1 -1
  118. package/static/app/assets/{hard-drive-D20iavUb.js β†’ hard-drive-DDzET7lk.js} +1 -1
  119. package/static/app/assets/index-CB93CZWW.css +2 -0
  120. package/static/app/assets/index-D2H-wSl6.js +13 -0
  121. package/static/app/assets/input-Df1CAY_I.js +1 -0
  122. package/static/app/assets/jsx-runtime-bzQ4Vb5N.js +1 -0
  123. package/static/app/assets/{link-2-BPJOFlAy.js β†’ link-2-xNnTIX1_.js} +1 -1
  124. package/static/app/assets/{permissionCopy-ChdJd493.js β†’ permissionCopy-D3aWHco-.js} +1 -1
  125. package/static/app/assets/primitives-BioD2slS.js +1 -0
  126. package/static/app/assets/search-BzBw8YcW.js +1 -0
  127. package/static/app/assets/{share-2-YNX_NtMU.js β†’ share-2-FkzGf8Df.js} +1 -1
  128. package/static/app/assets/{shield-alert-DuQ3zrVL.js β†’ shield-alert-B3dwzik4.js} +1 -1
  129. package/static/app/assets/sourceMeta-DQSY_tah.js +1 -0
  130. package/static/app/assets/textarea-P8o6pvOP.js +1 -0
  131. package/static/app/assets/useFocusTrap-hswOIkXE.js +1 -0
  132. package/static/app/assets/useMutation-OJLrYSRA.js +1 -0
  133. package/static/app/assets/workspace-BCuk3Ku9.js +1 -0
  134. package/static/app/index.html +4 -4
  135. package/static/sw.js +1 -1
  136. package/lattice_brain/ingestion/pipeline.py +0 -108
  137. package/latticeai/api/local_files.py +0 -44
  138. package/latticeai/api/tools.py +0 -126
  139. package/latticeai/api/voice_capture.py +0 -32
  140. package/latticeai/core/agent_permission.py +0 -85
  141. package/scripts/agent_eval.py +0 -34
  142. package/scripts/brain_quality_eval.py +0 -37
  143. package/scripts/check_legacy_debt.mjs +0 -91
  144. package/scripts/check_python.py +0 -100
  145. package/scripts/chunking_parity_corpus.py +0 -449
  146. package/scripts/generate_agent_parity_fixtures.py +0 -771
  147. package/scripts/generate_chunking_parity_fixtures.py +0 -259
  148. package/static/app/assets/Act-BPcVAbOL.js +0 -1
  149. package/static/app/assets/AdminConsole-Bw1ATQL0.js +0 -1
  150. package/static/app/assets/Brain-CT92Kos0.js +0 -321
  151. package/static/app/assets/BrainHome-CFBkt1K_.js +0 -2
  152. package/static/app/assets/BrainSignals-ReLWF2H8.js +0 -1
  153. package/static/app/assets/Capture-BsTokYkk.js +0 -1
  154. package/static/app/assets/Chronicle-B6f0T9id.js +0 -1
  155. package/static/app/assets/CommandPalette-CuvjTv1u.js +0 -1
  156. package/static/app/assets/Library-BGJbG9Hd.js +0 -1
  157. package/static/app/assets/LivingBrain-DGYK_Jsa.js +0 -1
  158. package/static/app/assets/ProductFlow-DXBC6brE.js +0 -1
  159. package/static/app/assets/System-CMHSO9qM.js +0 -1
  160. package/static/app/assets/arrow-left-BfmkskWx.js +0 -1
  161. package/static/app/assets/brain-CQJberbE.js +0 -1
  162. package/static/app/assets/button-Ct9f2_oT.js +0 -1
  163. package/static/app/assets/circle-check-DruOxB-4.js +0 -1
  164. package/static/app/assets/index-D9x-kSNy.css +0 -2
  165. package/static/app/assets/index-Do83hDzJ.js +0 -10
  166. package/static/app/assets/input-BLXVNmj1.js +0 -1
  167. package/static/app/assets/primitives-Cv5tbZBY.js +0 -1
  168. package/static/app/assets/search-CT9aho2j.js +0 -1
  169. package/static/app/assets/textarea-DqwLnli4.js +0 -1
  170. package/static/app/assets/useFocusTrap-ZVI98jaW.js +0 -1
  171. package/static/app/assets/useMutation-CVC4qv_D.js +0 -1
  172. package/static/app/assets/useQuery-C7BeG4HU.js +0 -1
  173. package/static/app/assets/utils-CiFtIdZq.js +0 -4
  174. package/static/app/assets/workspace-DQz9vIId.js +0 -1
@@ -64,6 +64,6 @@ def parse_model_ref(model_id: str) -> tuple[str, str]:
64
64
  return provider, model
65
65
  if provider in {"local_mlx", "mlx"}:
66
66
  return "local_mlx", model
67
- if model_id.startswith("local_mlx:"):
68
- return "local_mlx", model_id.split(":", 1)[1] # pragma: no cover β€” dead: a "local_mlx:" ref always has a ":" and returned above
67
+ # No trailing ``local_mlx:`` case: such a ref contains a colon, so it was
68
+ # already answered by the branch above.
69
69
  return "local_mlx", model_id
@@ -10,7 +10,7 @@ mistaken for the model's answer.
10
10
  import asyncio
11
11
  import base64
12
12
  import io
13
- from typing import Any, AsyncIterator, Optional
13
+ from typing import Any, AsyncIterator, List, Optional
14
14
 
15
15
  from PIL import Image
16
16
 
@@ -18,17 +18,105 @@ from latticeai.core.quiet import quiet
18
18
 
19
19
  from ._contract import RouterCore as _Core
20
20
  from .branding import SYSTEM_PROMPT, _compose_system, normalize_branding
21
+
22
+
23
+ def _system_for(context, cite_sources: bool) -> str:
24
+ """The system prompt for one completion.
25
+
26
+ ``cite_sources`` is what tells the two kinds of **caller** apart, and
27
+ v12.0.0 added it because nothing did.
28
+
29
+ A *chat* caller sends retrieved passages, and gets this product's own
30
+ system prompt with the Context block and
31
+ :data:`~latticeai.models.router.branding.CITATION_INSTRUCTION` appended.
32
+ :func:`_compose_system` used to do that for any non-empty context, which is
33
+ right there and wrong for the agent seam, whose context is the loop's own
34
+ executor prompt: every agent turn was being told its instructions were
35
+ "retrieved sources" to "cite inline as [1], [2]". A large model ignores
36
+ that. A small one obeys it β€” the acid-test 0.5B wrote
37
+ ``[1] 인사말을 μ“Έ μ˜ˆμ‹œ μ½”λ“œ: …`` into a file, and a 2B answered a tool call
38
+ with the citation instruction itself.
39
+
40
+ A *worker* caller sends **its own whole prompt**, and gets exactly that.
41
+ Same fact, one flag: a context that is not a corpus is an instruction, and
42
+ an instruction the caller wrote is the only instruction that turn should
43
+ carry. Until now the chat persona was still prepended to it, so six lines
44
+ of "You are Lattice AI … You are a Vision-Language Model … Be concise" sat
45
+ in front of every guided micro-turn β€” including the one that asks a model to
46
+ write a file's contents. Small models answer the nearest instruction: a live
47
+ 2B asked to summarise a README wrote "I am a local AI assistant that can run
48
+ on Apple Silicon" into the file, and a gemma-4-e2b opened two of its three
49
+ files with "Identity: Lattice AI (Vision-Language Model on Apple Silicon)".
50
+ Nothing leaked verbatim; the *subject* leaked, which is the same defect one
51
+ paraphrase further on. The document path has always worked this way β€” its
52
+ own system prompt replaces the chat identity entirely β€” so this is the
53
+ existing rule reaching the second caller that has a prompt of its own.
54
+
55
+ Both agent prompts still say who they are (``You are the executor of an
56
+ agent loop``) and every guided block still carries the run's own answer
57
+ language, so nothing the loop depends on is being removed β€” only a second,
58
+ competing identity. Branding normalisation is untouched: it runs over the
59
+ context here and over every generated string on the way out.
60
+
61
+ A worker call with **no** context still gets the product prompt. A bare
62
+ completion carrying no instruction at all is the one case where the identity
63
+ is the only thing there is to say.
64
+ """
65
+ context = normalize_branding(context)
66
+ if cite_sources:
67
+ return _compose_system(SYSTEM_PROMPT, context)
68
+ return context or SYSTEM_PROMPT
21
69
  from .catalog import CloudModel
22
70
  from .errors import ModelStreamError, _stream_failure
23
- from .loading import _mlx_sampler, executor
71
+ from .loading import _mlx_sampler, apply_prefix, apply_stop_strings, executor
72
+
73
+
74
+ def _stream_until_stop(
75
+ model,
76
+ tokenizer,
77
+ prompt: str,
78
+ image,
79
+ max_tokens: int,
80
+ sampler,
81
+ draft_model,
82
+ use_vlm: bool,
83
+ stops: List[str],
84
+ ) -> str:
85
+ """Generate on the worker thread, ending at the first stop string.
86
+
87
+ The buffered backends have no stop-string argument, so honouring one means
88
+ driving the *streaming* backend and breaking out of it. That is the whole
89
+ difference between a stop string and trimming the answer afterwards: the
90
+ tokens past the stop are never produced, which is time a 2B model would
91
+ otherwise spend explaining a tool call it has already emitted.
92
+
93
+ Runs inside :data:`executor`, like every other MLX call, so the GPU stream
94
+ stays on one thread.
95
+ """
96
+ if use_vlm:
97
+ from mlx_vlm import stream_generate as vlm_stream
98
+
99
+ chunks = vlm_stream(model, tokenizer, prompt=prompt, image=image, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model, draft_kind="mtp")
100
+ else:
101
+ from mlx_lm import stream_generate as lm_stream
102
+
103
+ chunks = lm_stream(model, tokenizer, prompt=prompt, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model)
104
+ text = ""
105
+ for chunk in chunks:
106
+ piece = chunk.text if hasattr(chunk, "text") else (chunk[0] if isinstance(chunk, tuple) else str(chunk))
107
+ text += piece
108
+ if any(marker in text for marker in stops):
109
+ break
110
+ return apply_stop_strings(text, stops)
24
111
 
25
112
 
26
113
  class _GenerationMixin(_Core):
27
114
  """The chat generation half of :class:`LLMRouter`."""
28
115
 
29
- def _build_prompt(self, message: str, context: Optional[str], tokenizer) -> str:
30
- context = normalize_branding(context)
31
- system = _compose_system(SYSTEM_PROMPT, context)
116
+ def _build_prompt(
117
+ self, message: str, context: Optional[str], tokenizer, cite_sources: bool = True
118
+ ) -> str:
119
+ system = _system_for(context, cite_sources)
32
120
  if hasattr(tokenizer, "apply_chat_template"):
33
121
  try:
34
122
  msgs = [{"role": "system", "content": system}, {"role": "user", "content": message}]
@@ -37,9 +125,8 @@ class _GenerationMixin(_Core):
37
125
  quiet()
38
126
  return f"<|im_start|>system\n{system}<|im_end|>\n<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
39
127
 
40
- def _build_vlm_prompt(self, model, processor, message: str, context: Optional[str], num_images: int) -> str:
41
- context = normalize_branding(context)
42
- system = _compose_system(SYSTEM_PROMPT, context)
128
+ def _build_vlm_prompt(self, model, processor, message: str, context: Optional[str], num_images: int, cite_sources: bool = True) -> str:
129
+ system = _system_for(context, cite_sources)
43
130
  try:
44
131
  from mlx_vlm import apply_chat_template
45
132
 
@@ -55,7 +142,7 @@ class _GenerationMixin(_Core):
55
142
  )
56
143
  except Exception as e:
57
144
  print(f"⚠️ VLM chat template fallback: {e}")
58
- return self._build_prompt(message, context, processor)
145
+ return self._build_prompt(message, context, processor, cite_sources)
59
146
 
60
147
  async def generate_as(
61
148
  self,
@@ -65,12 +152,35 @@ class _GenerationMixin(_Core):
65
152
  max_tokens: int = 4096,
66
153
  temperature: float = 0.2,
67
154
  image_data: Optional[str] = None,
155
+ stop: Optional[List[str]] = None,
156
+ prefix: Optional[str] = None,
157
+ cite_sources: bool = True,
68
158
  ) -> str:
69
- """Generate with a request-scoped model without changing the default."""
159
+ """Generate with a request-scoped model without changing the default.
160
+
161
+ ``stop`` ends the reply at the first of the given strings. Local
162
+ generation switches to the streaming backend to do it, so the tokens
163
+ after the stop are never produced rather than merely trimmed; the cloud
164
+ path forwards the list to the provider, which stops server-side. Absent
165
+ or empty means what it always meant β€” generate to ``max_tokens``.
166
+
167
+ ``prefix`` **starts** the reply (v12.0.0): the characters are put in the
168
+ model's mouth rather than requested of it, so a caller that needs the
169
+ answer to begin ``{"thoughts": "`` gets that by construction instead of
170
+ by asking nicely and repairing the result. Locally this is a real
171
+ prefill β€” the text is appended to the templated prompt after the
172
+ generation marker, so the model continues from mid-token rather than
173
+ starting a fresh turn. The returned text always begins with it; see
174
+ :func:`~latticeai.models.router.loading.apply_prefix` for how the three
175
+ backends are reconciled.
176
+ """
70
177
  _selected, cached = self._model_snapshot(model_id)
71
178
  if cached is None:
72
179
  return "No model."
73
- return await self._generate_cached(cached, message, context, max_tokens, temperature, image_data)
180
+ return await self._generate_cached(
181
+ cached, message, context, max_tokens, temperature, image_data, stop, prefix,
182
+ cite_sources,
183
+ )
74
184
 
75
185
  async def generate(
76
186
  self,
@@ -79,8 +189,14 @@ class _GenerationMixin(_Core):
79
189
  max_tokens: int = 4096,
80
190
  temperature: float = 0.2,
81
191
  image_data: Optional[str] = None,
192
+ stop: Optional[List[str]] = None,
193
+ prefix: Optional[str] = None,
194
+ cite_sources: bool = True,
82
195
  ) -> str:
83
- return await self.generate_as(None, message, context, max_tokens, temperature, image_data)
196
+ return await self.generate_as(
197
+ None, message, context, max_tokens, temperature, image_data, stop, prefix,
198
+ cite_sources,
199
+ )
84
200
 
85
201
  async def _generate_cached(
86
202
  self,
@@ -90,51 +206,81 @@ class _GenerationMixin(_Core):
90
206
  max_tokens: int,
91
207
  temperature: float,
92
208
  image_data: Optional[str],
209
+ stop: Optional[List[str]] = None,
210
+ prefix: Optional[str] = None,
211
+ cite_sources: bool = True,
93
212
  ) -> str:
94
213
  if isinstance(cached, CloudModel):
95
- return await self._cloud_generate(cached, message, context, max_tokens, temperature)
214
+ return await self._cloud_generate(
215
+ cached, message, context, max_tokens, temperature, stop, prefix, cite_sources
216
+ )
96
217
 
97
218
  model, tokenizer, draft_model, loader_kind = self._unpack_local_cache(cached)
98
219
  use_vlm = loader_kind == "mlx_vlm"
99
220
  prompt = (
100
- self._build_vlm_prompt(model, tokenizer, message, context, 1 if image_data else 0)
221
+ self._build_vlm_prompt(
222
+ model, tokenizer, message, context, 1 if image_data else 0, cite_sources
223
+ )
101
224
  if use_vlm
102
- else self._build_prompt(message, context, tokenizer)
225
+ else self._build_prompt(message, context, tokenizer, cite_sources)
103
226
  )
104
-
227
+ # The prefill. Appended *after* the chat template's generation marker,
228
+ # so the first sampled token continues these characters instead of
229
+ # opening a reply. Nothing else in the pipeline changes.
230
+ if prefix:
231
+ prompt = f"{prompt}{prefix}"
232
+ stops = [marker for marker in (stop or []) if marker]
233
+
105
234
  loop = asyncio.get_event_loop()
106
-
235
+
107
236
  def _gen():
108
237
  import mlx.core as mx # type: ignore[no-redef]
109
238
 
110
239
  mx.set_default_device(mx.gpu) # type: ignore[arg-type]
240
+ # Decoded here, not above: base64 + PIL is CPU work, and this
241
+ # function is the part that runs off the event loop.
242
+ image = self._prep_image(image_data) if image_data else None
243
+ sampler = _mlx_sampler(temperature)
244
+ if stops:
245
+ return _stream_until_stop(
246
+ model, tokenizer, prompt, image, max_tokens, sampler, draft_model,
247
+ use_vlm, stops,
248
+ )
111
249
  if use_vlm:
112
250
  from mlx_vlm import generate as vlm_gen
113
- return vlm_gen(model, tokenizer, prompt=prompt, image=self._prep_image(image_data) if image_data else None, max_tokens=max_tokens, sampler=_mlx_sampler(temperature), draft_model=draft_model, draft_kind="mtp")
251
+ return vlm_gen(model, tokenizer, prompt=prompt, image=image, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model, draft_kind="mtp")
114
252
  from mlx_lm import generate as lm_gen
115
- return lm_gen(model, tokenizer, prompt=prompt, max_tokens=max_tokens, sampler=_mlx_sampler(temperature), draft_model=draft_model)
253
+ return lm_gen(model, tokenizer, prompt=prompt, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model)
116
254
  result = await loop.run_in_executor(executor, _gen)
117
255
  # mlx-vlm might return a GenerationResult object; extract the text
118
- if hasattr(result, "text"):
119
- return normalize_branding(result.text)
120
- return normalize_branding(str(result))
256
+ text = result.text if hasattr(result, "text") else str(result)
257
+ return apply_prefix(prefix, normalize_branding(apply_stop_strings(text, stops)))
121
258
 
122
- async def _cloud_generate(self, cloud: CloudModel, message: str, context: Optional[str], max_tokens: int, temperature: float) -> str:
123
- context = normalize_branding(context)
124
- system = _compose_system(SYSTEM_PROMPT, context)
259
+ async def _cloud_generate(self, cloud: CloudModel, message: str, context: Optional[str], max_tokens: int, temperature: float, stop: Optional[List[str]] = None, prefix: Optional[str] = None, cite_sources: bool = True) -> str:
260
+ system = _system_for(context, cite_sources)
261
+ stops = [marker for marker in (stop or []) if marker]
262
+ extra = {"stop": stops} if stops else {}
263
+ messages = [
264
+ {"role": "system", "content": system},
265
+ {"role": "user", "content": message},
266
+ ]
267
+ # Assistant prefill: the OpenAI-compatible way to say "continue this".
268
+ # Local servers (vLLM, llama.cpp, LM Studio, Ollama) honour it; a
269
+ # provider that does not simply reads it as context, and `apply_prefix`
270
+ # makes both answers the same shape for the caller.
271
+ if prefix:
272
+ messages.append({"role": "assistant", "content": prefix})
125
273
  try:
126
274
  response = await cloud.client.chat.completions.create(
127
275
  model=cloud.model,
128
- messages=[
129
- {"role": "system", "content": system},
130
- {"role": "user", "content": message},
131
- ],
276
+ messages=messages,
132
277
  max_tokens=max_tokens,
133
278
  temperature=temperature,
279
+ **extra,
134
280
  )
135
281
  except Exception as e:
136
282
  raise RuntimeError(self._local_server_error_hint(cloud, e)) from e
137
- return normalize_branding(response.choices[0].message.content or "")
283
+ return apply_prefix(prefix, normalize_branding(response.choices[0].message.content or ""))
138
284
 
139
285
  async def stream_generate_as(
140
286
  self,
@@ -25,6 +25,7 @@ import gc
25
25
  from concurrent.futures import ThreadPoolExecutor
26
26
  from typing import Any, Dict, List, Optional
27
27
 
28
+ from latticeai.core.quiet import quiet
28
29
  from latticeai.models.model_providers import (
29
30
  OPENAI_COMPATIBLE_PROVIDERS,
30
31
  PROVIDER_MODEL_CATALOG,
@@ -120,12 +121,145 @@ def ensure_mlx_runtime() -> None:
120
121
  def _mlx_sampler(temperature: float):
121
122
  """Build an MLX sampler callable for the given temperature.
122
123
 
123
- Lattice v2.2 keeps local execution on MLX-VLM only. Returning ``None`` lets
124
- MLX-VLM use its bundled default sampler without pulling another generation
125
- package into the runtime contract.
124
+ Until v11.9.0 this discarded its argument and returned ``None``, which let
125
+ MLX pick its own default sampler. The cost was invisible and large: the
126
+ agent loop asks for 0.1 when it wants a tool call and 0.0 on the strict
127
+ verification re-ask, and **locally neither did anything**. A cloud model got
128
+ the deterministic re-ask it was designed around; the 2B model on this
129
+ machine got the same creative sampler it had on the first attempt, and the
130
+ "one strict retry" was one more roll of the same dice.
131
+
132
+ ``mlx_lm.sample_utils.make_sampler`` is the sampler both backends accept β€”
133
+ ``mlx_vlm`` takes the callable straight through β€” and it is already an
134
+ installed dependency of the text path. If it cannot be imported (an
135
+ MLX-VLM-only install, a version that moved it) the answer is the old one:
136
+ ``None``, the backend's default. A missing sampler must never be a failed
137
+ generation.
126
138
  """
127
- _ = temperature
128
- return
139
+ try:
140
+ from mlx_lm.sample_utils import make_sampler
141
+ except Exception: # noqa: BLE001 β€” an absent sampler is not a failure
142
+ quiet("MLX sampler unavailable; using the backend default")
143
+ return None
144
+ try:
145
+ return make_sampler(temp=float(temperature))
146
+ except Exception: # noqa: BLE001 β€” a signature change must not stop a run
147
+ quiet("MLX sampler construction failed; using the backend default")
148
+ return None
149
+
150
+
151
+ def _declares_vision(model_id: str) -> bool:
152
+ """Whether this checkpoint's own config says it is multimodal.
153
+
154
+ Read from ``config.json`` rather than decided from the model's *name*: a
155
+ list of known vision families is a list that is wrong about every model
156
+ published after it was written, and "which models work" is exactly the
157
+ question this product must not answer with a hardcoded roster.
158
+
159
+ The three markers are the ones every MLX-VLM architecture carries β€” a
160
+ nested ``vision_config``, a ``vision_tower``, or an ``image_token_index``
161
+ for the placeholder the processor substitutes. A config that carries none
162
+ of them describes a text model.
163
+ """
164
+ import json as _json
165
+ from pathlib import Path as _Path
166
+
167
+ from .local_models import hf_model_dir
168
+
169
+ raw = str(model_id or "").strip()
170
+ candidates = []
171
+ explicit = _Path(raw).expanduser()
172
+ if raw and explicit.exists():
173
+ candidates.append(explicit / "config.json")
174
+ try:
175
+ candidates.append(hf_model_dir(raw) / "config.json")
176
+ except Exception as error: # noqa: BLE001 β€” an unresolvable id is not a claim
177
+ quiet(f"vision probe skipped for {raw}: {error}")
178
+ for config_path in candidates:
179
+ try:
180
+ if not config_path.exists():
181
+ continue
182
+ data = _json.loads(config_path.read_text(encoding="utf-8"))
183
+ except Exception as error: # noqa: BLE001 β€” an unreadable config is not a claim
184
+ quiet(f"vision probe skipped for {config_path}: {error}")
185
+ continue
186
+ if any(key in data for key in ("vision_config", "vision_tower", "image_token_index")):
187
+ return True
188
+ return False
189
+
190
+
191
+ def _text_fallback_allowed(
192
+ model_id: str, target_model_id: str, model_type: Optional[str]
193
+ ) -> bool:
194
+ """Whether a failed multimodal load may be retried as a text load.
195
+
196
+ Every local model is routed to MLX-VLM first, because the recommended
197
+ defaults are multimodal. Until v12.0.0 the retry on the text path was
198
+ gated on the model *id* matching Gemma 4 β€” so a plain text model MLX-VLM
199
+ cannot open (a 0.5B Qwen, an AWQ checkpoint, any fine-tune with a text-only
200
+ architecture) failed to load at all, with MLX-LM sitting right there able
201
+ to open it. That is a roster deciding what runs, and the roster was one
202
+ family long. "Every small model works" cannot be true while loading is
203
+ gated on a name.
204
+
205
+ So the rule is now two rules, and the second is **additive**:
206
+
207
+ * the v11.9.0 Gemma 4 rule, unchanged β€” a Gemma 4 that is not the unified
208
+ (vision) architecture may always retry as text;
209
+ * and, new, **any model whose own ``config.json`` does not declare
210
+ vision.** Read from the checkpoint rather than decided from the name, so
211
+ a model published tomorrow is judged the same way as one published last
212
+ year.
213
+
214
+ A config that *does* declare vision is never silently downgraded: answering
215
+ an image request from a text-only load is worse than refusing it.
216
+ """
217
+ if lm_load is None:
218
+ return False
219
+ if (model_type or "").strip().lower() == "gemma4_unified":
220
+ return False
221
+ if _is_gemma4_model_id(model_id):
222
+ return True
223
+ return not _declares_vision(target_model_id)
224
+
225
+
226
+ def apply_prefix(prefix: Optional[str], text: str) -> str:
227
+ """Guarantee ``text`` begins with ``prefix``, without doubling it.
228
+
229
+ The seam's contract for a forced prefix (v12.0.0) is one sentence: *the
230
+ reply starts with these characters*. How it got there differs by backend β€”
231
+ the local path really does prefill the prompt with them, so the model never
232
+ generated them and they have to be put back; a cloud provider handed an
233
+ assistant prefill message may echo them, may not, and may ignore the
234
+ message entirely. One normalisation covers all three, so the caller parses
235
+ one shape whichever answered.
236
+
237
+ Leading whitespace is dropped when the reply already carries the prefix:
238
+ a chat template that emits ``"\\n"`` before the continuation would otherwise
239
+ make ``startswith`` false for a reply that plainly does start with it.
240
+ """
241
+ if not prefix:
242
+ return text
243
+ stripped = text.lstrip()
244
+ if stripped.startswith(prefix):
245
+ return stripped
246
+ return f"{prefix}{text}"
247
+
248
+
249
+ def apply_stop_strings(text: str, stop: Optional[List[str]]) -> str:
250
+ """Cut ``text`` at the earliest stop string, if any is present.
251
+
252
+ Shared by the buffered and streaming local paths so "where does this reply
253
+ end" has one answer. Empty stop strings are ignored: a caller that sends
254
+ ``""`` means "no stop", not "stop before the first character".
255
+ """
256
+ if not stop:
257
+ return text
258
+ cut = min(
259
+ (text.index(marker) for marker in stop if marker and marker in text),
260
+ default=-1,
261
+ )
262
+ return text if cut < 0 else text[:cut]
129
263
 
130
264
 
131
265
  class _LoadingMixin(_Core):
@@ -170,7 +304,6 @@ class _LoadingMixin(_Core):
170
304
 
171
305
  def _load():
172
306
  mx.set_default_device(mx.gpu)
173
- is_gemma4 = _is_gemma4_model_id(model_id)
174
307
  model_type = _local_model_type(target_model_id) or _local_model_type(model_id)
175
308
  loader_kind = "mlx_vlm"
176
309
 
@@ -180,11 +313,19 @@ class _LoadingMixin(_Core):
180
313
  print(f"πŸ”„ Loading Target (VLM Mode): {target_model_id}...")
181
314
  model, tokenizer = vlm_load(target_model_id)
182
315
  except Exception as vlm_error:
183
- if not (is_gemma4 and model_type != "gemma4_unified" and lm_load is not None):
316
+ if not _text_fallback_allowed(model_id, target_model_id, model_type):
184
317
  raise
185
- print(f"⚠️ Gemma 4 MLX-VLM load failed; retrying MLX-LM text path: {vlm_error}")
318
+ print(f"⚠️ MLX-VLM load failed; retrying the MLX-LM text path: {vlm_error}")
186
319
  print(f"πŸ”„ Loading Target (LM Mode): {target_model_id}...")
187
- model, tokenizer = lm_load(target_model_id)
320
+ try:
321
+ model, tokenizer = lm_load(target_model_id)
322
+ except Exception:
323
+ # The text path is a *fallback*: when it fails too, the
324
+ # useful diagnosis is the first failure, from the loader
325
+ # this model was routed to. Raising the second one would
326
+ # tell an operator their multimodal model is not a text
327
+ # model, which they knew.
328
+ raise vlm_error from None
188
329
  loader_kind = "mlx_lm"
189
330
 
190
331
  draft_model = None
@@ -45,8 +45,10 @@ def build_access_runtime(
45
45
  # extend this trust to public or non-loopback bindings, even if an invalid
46
46
  # caller constructs this runtime with ``require_auth=False`` directly.
47
47
  externally_reachable = is_externally_reachable(config)
48
+ # One flag, not two: "authentication is required" is exactly
49
+ # ``not trusted_local_owner``, so deriving it separately only creates a
50
+ # pair that a later edit can put out of step.
48
51
  trusted_local_owner = not require_auth and not externally_reachable
49
- effective_require_auth = bool(require_auth or externally_reachable)
50
52
 
51
53
  def get_user_role(email: str, users: Optional[Dict] = None) -> str:
52
54
  users = users or load_users()
@@ -128,9 +130,10 @@ def build_access_runtime(
128
130
  # shared local vaults, and Local User profile behavior compatible;
129
131
  # get_user_role() supplies the explicit owner authorization role.
130
132
  return ""
131
- if effective_require_auth and not email:
132
- raise http_exception(status_code=401, detail="인증이 ν•„μš”ν•©λ‹ˆλ‹€.")
133
- return email or "" # pragma: no cover β€” unreachable: trusted_local_owner is the exact complement of effective_require_auth
133
+ # Reaching here means the caller is not the trusted local owner and no
134
+ # session produced an email β€” i.e. authentication is required and was
135
+ # not supplied. There is no third outcome to fall through to.
136
+ raise http_exception(status_code=401, detail="인증이 ν•„μš”ν•©λ‹ˆλ‹€.")
134
137
 
135
138
  def require_admin(request: request_type) -> tuple[str, Dict]:
136
139
  users = load_users()
@@ -13,7 +13,16 @@ was requested.
13
13
 
14
14
  from __future__ import annotations
15
15
 
16
- from typing import Any, Callable, Mapping
16
+ from typing import Any, Callable, Mapping, Optional, Tuple
17
+
18
+ from latticeai.core.embedding_providers.autodetect import (
19
+ AUTO_PROVIDER,
20
+ Detection,
21
+ detect_embedder,
22
+ resolve_auto_provider,
23
+ )
24
+
25
+ _FALLBACK_NAMES = {"", "hash", "local", "fallback"}
17
26
 
18
27
 
19
28
  def build_embedder_runtime(
@@ -21,16 +30,26 @@ def build_embedder_runtime(
21
30
  config: Any,
22
31
  profile: Mapping[str, Any],
23
32
  resolve_embedder: Callable[..., Any],
33
+ detect: Optional[Callable[..., Detection]] = None,
24
34
  ) -> Any:
25
- """Resolve the configured embedding provider once."""
35
+ """Resolve the configured embedding provider once.
26
36
 
27
- provider = config.embedding_provider
28
- model = config.embedding_model or str(profile.get("model") or "")
29
- dim = config.embedding_dim or int(profile.get("dimensions") or 0)
30
- if config.embedding_profile and provider in {"", "hash", "local", "fallback"}:
31
- provider = str(profile.get("provider") or provider)
37
+ The resolution order is the product's, unchanged, with one addition at the
38
+ end: detection. What is on this machine is *always* looked up (so
39
+ ``GET /api/embeddings/status`` can say "a real embedder is right there"),
40
+ and it changes the resolution only when the operator asked for it with
41
+ ``LATTICEAI_EMBEDDING_PROVIDER=auto``. The finding rides back on the
42
+ resolved object as ``detected`` β€” an added attribute, so every existing
43
+ reader of ``ResolvedEmbedder`` is untouched.
44
+ """
45
+
46
+ provider, model, dim = _configured(config, profile)
47
+ detection = (detect or detect_embedder)(
48
+ configured_provider=provider, configured_model=model
49
+ )
50
+ provider, model, dim = resolve_auto_provider(provider, model, dim, detection)
32
51
 
33
- return resolve_embedder(
52
+ resolved = resolve_embedder(
34
53
  provider,
35
54
  model=model,
36
55
  base_url=config.embedding_base_url,
@@ -38,8 +57,23 @@ def build_embedder_runtime(
38
57
  dim=dim,
39
58
  timeout=config.embedding_timeout,
40
59
  extra={"target": config.embedding_custom_target},
41
- probe=provider not in {"", "hash", "local", "fallback"},
60
+ probe=provider not in _FALLBACK_NAMES,
42
61
  )
62
+ try:
63
+ resolved.detected = detection
64
+ except AttributeError: # pragma: no cover - a stand-in without __dict__
65
+ pass
66
+ return resolved
67
+
68
+
69
+ def _configured(config: Any, profile: Mapping[str, Any]) -> Tuple[str, str, int]:
70
+ """``(provider, model, dim)`` from configuration and the named profile."""
71
+ provider = config.embedding_provider
72
+ model = config.embedding_model or str(profile.get("model") or "")
73
+ dim = config.embedding_dim or int(profile.get("dimensions") or 0)
74
+ if config.embedding_profile and provider in _FALLBACK_NAMES | {AUTO_PROVIDER}:
75
+ provider = str(profile.get("provider") or provider)
76
+ return provider, model, dim
43
77
 
44
78
 
45
79
  __all__ = ["build_embedder_runtime"]