ltcai 11.7.0 β 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -76
- package/docs/BENCHMARKS.md +9 -2
- package/docs/CHANGELOG.md +249 -0
- package/docs/CI_AND_RELEASE_GATES.md +126 -41
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +271 -103
- package/docs/ENTERPRISE.md +1 -1
- package/docs/LEGACY_COMPATIBILITY.md +10 -6
- package/docs/MULTI_AGENT_RUNTIME.md +4 -4
- package/docs/ONBOARDING.md +16 -4
- package/docs/OPERATIONS.md +14 -1
- package/docs/PERMISSION_MODE.md +14 -9
- package/docs/REALTIME_COLLABORATION.md +1 -1
- package/docs/ROADMAP.md +113 -0
- package/docs/TRUST_MODEL.md +28 -7
- package/docs/USABILITY_AUDIT.md +5 -0
- package/docs/WHY_LATTICE.md +13 -5
- package/docs/WORKFLOW_DESIGNER.md +2 -2
- package/docs/kg-schema.md +57 -7
- package/docs/mcp-tools.md +93 -82
- package/docs/security-model.md +6 -3
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/graph/_kg_common/__init__.py +1 -54
- package/lattice_brain/graph/_kg_common/extraction.py +459 -105
- package/lattice_brain/graph/_kg_common/normalize.py +305 -0
- package/lattice_brain/graph/_kg_common/patterns.py +275 -0
- package/lattice_brain/graph/_kg_common/relations.py +12 -3
- package/lattice_brain/graph/_kg_common/sections.py +107 -0
- package/lattice_brain/graph/_kg_common/text.py +14 -450
- package/lattice_brain/graph/_kg_constants.py +7 -0
- package/lattice_brain/ingestion/__init__.py +6 -3
- package/lattice_brain/multimodal/__init__.py +9 -3
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/agent_worker_seam.py +44 -1
- package/latticeai/api/models.py +18 -110
- package/latticeai/api/search.py +7 -30
- package/latticeai/api/worker_compute.py +127 -106
- package/latticeai/api/worker_seams.py +17 -2
- package/latticeai/core/embedding_providers/__init__.py +16 -0
- package/latticeai/core/embedding_providers/autodetect.py +302 -0
- package/latticeai/core/embedding_providers/base.py +25 -0
- package/latticeai/core/embedding_providers/profiles.py +44 -0
- package/latticeai/core/embedding_providers/text.py +74 -8
- package/latticeai/core/http_origin.py +3 -3
- package/latticeai/core/messages.py +0 -5
- package/latticeai/core/policy.py +1 -6
- package/latticeai/core/quiet.py +1 -20
- package/latticeai/core/security.py +29 -83
- package/latticeai/core/sessions.py +95 -4
- package/latticeai/core/users.py +0 -38
- package/latticeai/core/vector_index/__init__.py +61 -0
- package/latticeai/core/vector_index/hnsw.py +383 -0
- package/latticeai/core/vector_index/sidecar.py +329 -0
- package/latticeai/models/router/catalog.py +2 -2
- package/latticeai/models/router/generation.py +176 -30
- package/latticeai/models/router/loading.py +150 -9
- package/latticeai/runtime/access_runtime.py +7 -4
- package/latticeai/runtime/brain_runtime.py +43 -9
- package/latticeai/runtime/build_phases/features.py +8 -31
- package/latticeai/runtime/build_phases/foundation.py +7 -16
- package/latticeai/runtime/build_phases/web.py +3 -3
- package/latticeai/runtime/build_phases/worker_profile.py +29 -27
- package/latticeai/runtime/runtime_context.py +0 -2
- package/latticeai/services/architecture_readiness.py +18 -19
- package/latticeai/services/process_audit.py +1 -22
- package/latticeai/services/product_readiness.py +39 -12
- package/latticeai/services/search_service.py +7 -0
- package/latticeai/services/voice_capture.py +8 -28
- package/latticeai/tools/__init__.py +12 -47
- package/latticeai/tools/commands.py +9 -15
- package/latticeai/tools/documents.py +12 -0
- package/latticeai/tools/knowledge.py +0 -6
- package/latticeai/tools/markup.py +152 -0
- package/package.json +4 -5
- package/requirements.txt +0 -1
- package/scripts/check_current_release_docs.mjs +1 -1
- package/scripts/check_openapi_drift.mjs +3 -2
- package/scripts/check_server_i18n.mjs +5 -4
- package/scripts/compose_openapi.py +4 -1
- package/scripts/export_openapi.py +5 -4
- package/scripts/gen_worker_allowlist_fixture.py +2 -2
- package/scripts/openapi_route_families.json +19 -74
- package/scripts/publish_release.mjs +157 -0
- package/scripts/release_screen_claims.json +144 -28
- package/src-tauri/Cargo.lock +45 -10
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +47 -41
- package/static/app/assets/Act-Cf1L2709.js +2 -0
- package/static/app/assets/AdminConsole-DPAbLTYV.js +1 -0
- package/static/app/assets/Brain-DqamGrj-.js +2 -0
- package/static/app/assets/BrainHome-MHe2_RYs.js +2 -0
- package/static/app/assets/BrainSignals-CQPPfyyH.js +1 -0
- package/static/app/assets/Capture-DGdIH_Zc.js +1 -0
- package/static/app/assets/Chronicle-C-UlCJoJ.js +1 -0
- package/static/app/assets/CommandPalette-WNT4EqUX.js +1 -0
- package/static/app/assets/DigitalBrainExplorer-CEBH5Cwc.js +321 -0
- package/static/app/assets/Library-C6xd1dlf.js +1 -0
- package/static/app/assets/LivingBrain-BEk-0ohw.js +1 -0
- package/static/app/assets/ProductFlow-CZLm5iXh.js +1 -0
- package/static/app/assets/QueryClientProvider-B3OjqSyJ.js +1 -0
- package/static/app/assets/{ReviewCard-HXRle3qq.js β ReviewCard-CEHG6evf.js} +2 -2
- package/static/app/assets/RunsListPanel-CLtEJSRW.js +1 -0
- package/static/app/assets/System-CAxwBUXw.js +1 -0
- package/static/app/assets/WorkflowGraph-Dj10RuGE.js +1 -0
- package/static/app/assets/WorkflowsPanel-Kyeh_LIT.js +2 -0
- package/static/app/assets/actHelpers-CtSmK9Dw.js +1 -0
- package/static/app/assets/arrow-left-CRl5EO4D.js +1 -0
- package/static/app/assets/{bot-Cn8bWRuq.js β bot-DhUGRel2.js} +1 -1
- package/static/app/assets/brain-CLkhHsHF.js +1 -0
- package/static/app/assets/button-CmaEqG1T.js +1 -0
- package/static/app/assets/circle-check-CFgejkOS.js +1 -0
- package/static/app/assets/{circle-pause-CmzC_apg.js β circle-pause-l96izbxj.js} +1 -1
- package/static/app/assets/{circle-play-D8mW2aQ7.js β circle-play-CrZa25_q.js} +1 -1
- package/static/app/assets/{cpu-DZcdd0PZ.js β cpu-BaXudqwl.js} +1 -1
- package/static/app/assets/{download-bv1KEPGQ.js β download-hCVFPiyc.js} +1 -1
- package/static/app/assets/{folder-open-d-Pip5gr.js β folder-open-CHL82Yp7.js} +1 -1
- package/static/app/assets/{hard-drive-D20iavUb.js β hard-drive-DDzET7lk.js} +1 -1
- package/static/app/assets/index-CB93CZWW.css +2 -0
- package/static/app/assets/index-D2H-wSl6.js +13 -0
- package/static/app/assets/input-Df1CAY_I.js +1 -0
- package/static/app/assets/jsx-runtime-bzQ4Vb5N.js +1 -0
- package/static/app/assets/{link-2-BPJOFlAy.js β link-2-xNnTIX1_.js} +1 -1
- package/static/app/assets/{permissionCopy-ChdJd493.js β permissionCopy-D3aWHco-.js} +1 -1
- package/static/app/assets/primitives-BioD2slS.js +1 -0
- package/static/app/assets/search-BzBw8YcW.js +1 -0
- package/static/app/assets/{share-2-YNX_NtMU.js β share-2-FkzGf8Df.js} +1 -1
- package/static/app/assets/{shield-alert-DuQ3zrVL.js β shield-alert-B3dwzik4.js} +1 -1
- package/static/app/assets/sourceMeta-DQSY_tah.js +1 -0
- package/static/app/assets/textarea-P8o6pvOP.js +1 -0
- package/static/app/assets/useFocusTrap-hswOIkXE.js +1 -0
- package/static/app/assets/useMutation-OJLrYSRA.js +1 -0
- package/static/app/assets/workspace-BCuk3Ku9.js +1 -0
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/lattice_brain/ingestion/pipeline.py +0 -108
- package/latticeai/api/local_files.py +0 -44
- package/latticeai/api/tools.py +0 -126
- package/latticeai/api/voice_capture.py +0 -32
- package/latticeai/core/agent_permission.py +0 -85
- package/scripts/agent_eval.py +0 -34
- package/scripts/brain_quality_eval.py +0 -37
- package/scripts/check_legacy_debt.mjs +0 -91
- package/scripts/check_python.py +0 -100
- package/scripts/chunking_parity_corpus.py +0 -449
- package/scripts/generate_agent_parity_fixtures.py +0 -771
- package/scripts/generate_chunking_parity_fixtures.py +0 -259
- package/static/app/assets/Act-BPcVAbOL.js +0 -1
- package/static/app/assets/AdminConsole-Bw1ATQL0.js +0 -1
- package/static/app/assets/Brain-CT92Kos0.js +0 -321
- package/static/app/assets/BrainHome-CFBkt1K_.js +0 -2
- package/static/app/assets/BrainSignals-ReLWF2H8.js +0 -1
- package/static/app/assets/Capture-BsTokYkk.js +0 -1
- package/static/app/assets/Chronicle-B6f0T9id.js +0 -1
- package/static/app/assets/CommandPalette-CuvjTv1u.js +0 -1
- package/static/app/assets/Library-BGJbG9Hd.js +0 -1
- package/static/app/assets/LivingBrain-DGYK_Jsa.js +0 -1
- package/static/app/assets/ProductFlow-DXBC6brE.js +0 -1
- package/static/app/assets/System-CMHSO9qM.js +0 -1
- package/static/app/assets/arrow-left-BfmkskWx.js +0 -1
- package/static/app/assets/brain-CQJberbE.js +0 -1
- package/static/app/assets/button-Ct9f2_oT.js +0 -1
- package/static/app/assets/circle-check-DruOxB-4.js +0 -1
- package/static/app/assets/index-D9x-kSNy.css +0 -2
- package/static/app/assets/index-Do83hDzJ.js +0 -10
- package/static/app/assets/input-BLXVNmj1.js +0 -1
- package/static/app/assets/primitives-Cv5tbZBY.js +0 -1
- package/static/app/assets/search-CT9aho2j.js +0 -1
- package/static/app/assets/textarea-DqwLnli4.js +0 -1
- package/static/app/assets/useFocusTrap-ZVI98jaW.js +0 -1
- package/static/app/assets/useMutation-CVC4qv_D.js +0 -1
- package/static/app/assets/useQuery-C7BeG4HU.js +0 -1
- package/static/app/assets/utils-CiFtIdZq.js +0 -4
- package/static/app/assets/workspace-DQz9vIId.js +0 -1
|
@@ -64,6 +64,6 @@ def parse_model_ref(model_id: str) -> tuple[str, str]:
|
|
|
64
64
|
return provider, model
|
|
65
65
|
if provider in {"local_mlx", "mlx"}:
|
|
66
66
|
return "local_mlx", model
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
# No trailing ``local_mlx:`` case: such a ref contains a colon, so it was
|
|
68
|
+
# already answered by the branch above.
|
|
69
69
|
return "local_mlx", model_id
|
|
@@ -10,7 +10,7 @@ mistaken for the model's answer.
|
|
|
10
10
|
import asyncio
|
|
11
11
|
import base64
|
|
12
12
|
import io
|
|
13
|
-
from typing import Any, AsyncIterator, Optional
|
|
13
|
+
from typing import Any, AsyncIterator, List, Optional
|
|
14
14
|
|
|
15
15
|
from PIL import Image
|
|
16
16
|
|
|
@@ -18,17 +18,105 @@ from latticeai.core.quiet import quiet
|
|
|
18
18
|
|
|
19
19
|
from ._contract import RouterCore as _Core
|
|
20
20
|
from .branding import SYSTEM_PROMPT, _compose_system, normalize_branding
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _system_for(context, cite_sources: bool) -> str:
|
|
24
|
+
"""The system prompt for one completion.
|
|
25
|
+
|
|
26
|
+
``cite_sources`` is what tells the two kinds of **caller** apart, and
|
|
27
|
+
v12.0.0 added it because nothing did.
|
|
28
|
+
|
|
29
|
+
A *chat* caller sends retrieved passages, and gets this product's own
|
|
30
|
+
system prompt with the Context block and
|
|
31
|
+
:data:`~latticeai.models.router.branding.CITATION_INSTRUCTION` appended.
|
|
32
|
+
:func:`_compose_system` used to do that for any non-empty context, which is
|
|
33
|
+
right there and wrong for the agent seam, whose context is the loop's own
|
|
34
|
+
executor prompt: every agent turn was being told its instructions were
|
|
35
|
+
"retrieved sources" to "cite inline as [1], [2]". A large model ignores
|
|
36
|
+
that. A small one obeys it β the acid-test 0.5B wrote
|
|
37
|
+
``[1] μΈμ¬λ§μ μΈ μμ μ½λ: β¦`` into a file, and a 2B answered a tool call
|
|
38
|
+
with the citation instruction itself.
|
|
39
|
+
|
|
40
|
+
A *worker* caller sends **its own whole prompt**, and gets exactly that.
|
|
41
|
+
Same fact, one flag: a context that is not a corpus is an instruction, and
|
|
42
|
+
an instruction the caller wrote is the only instruction that turn should
|
|
43
|
+
carry. Until now the chat persona was still prepended to it, so six lines
|
|
44
|
+
of "You are Lattice AI β¦ You are a Vision-Language Model β¦ Be concise" sat
|
|
45
|
+
in front of every guided micro-turn β including the one that asks a model to
|
|
46
|
+
write a file's contents. Small models answer the nearest instruction: a live
|
|
47
|
+
2B asked to summarise a README wrote "I am a local AI assistant that can run
|
|
48
|
+
on Apple Silicon" into the file, and a gemma-4-e2b opened two of its three
|
|
49
|
+
files with "Identity: Lattice AI (Vision-Language Model on Apple Silicon)".
|
|
50
|
+
Nothing leaked verbatim; the *subject* leaked, which is the same defect one
|
|
51
|
+
paraphrase further on. The document path has always worked this way β its
|
|
52
|
+
own system prompt replaces the chat identity entirely β so this is the
|
|
53
|
+
existing rule reaching the second caller that has a prompt of its own.
|
|
54
|
+
|
|
55
|
+
Both agent prompts still say who they are (``You are the executor of an
|
|
56
|
+
agent loop``) and every guided block still carries the run's own answer
|
|
57
|
+
language, so nothing the loop depends on is being removed β only a second,
|
|
58
|
+
competing identity. Branding normalisation is untouched: it runs over the
|
|
59
|
+
context here and over every generated string on the way out.
|
|
60
|
+
|
|
61
|
+
A worker call with **no** context still gets the product prompt. A bare
|
|
62
|
+
completion carrying no instruction at all is the one case where the identity
|
|
63
|
+
is the only thing there is to say.
|
|
64
|
+
"""
|
|
65
|
+
context = normalize_branding(context)
|
|
66
|
+
if cite_sources:
|
|
67
|
+
return _compose_system(SYSTEM_PROMPT, context)
|
|
68
|
+
return context or SYSTEM_PROMPT
|
|
21
69
|
from .catalog import CloudModel
|
|
22
70
|
from .errors import ModelStreamError, _stream_failure
|
|
23
|
-
from .loading import _mlx_sampler, executor
|
|
71
|
+
from .loading import _mlx_sampler, apply_prefix, apply_stop_strings, executor
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _stream_until_stop(
|
|
75
|
+
model,
|
|
76
|
+
tokenizer,
|
|
77
|
+
prompt: str,
|
|
78
|
+
image,
|
|
79
|
+
max_tokens: int,
|
|
80
|
+
sampler,
|
|
81
|
+
draft_model,
|
|
82
|
+
use_vlm: bool,
|
|
83
|
+
stops: List[str],
|
|
84
|
+
) -> str:
|
|
85
|
+
"""Generate on the worker thread, ending at the first stop string.
|
|
86
|
+
|
|
87
|
+
The buffered backends have no stop-string argument, so honouring one means
|
|
88
|
+
driving the *streaming* backend and breaking out of it. That is the whole
|
|
89
|
+
difference between a stop string and trimming the answer afterwards: the
|
|
90
|
+
tokens past the stop are never produced, which is time a 2B model would
|
|
91
|
+
otherwise spend explaining a tool call it has already emitted.
|
|
92
|
+
|
|
93
|
+
Runs inside :data:`executor`, like every other MLX call, so the GPU stream
|
|
94
|
+
stays on one thread.
|
|
95
|
+
"""
|
|
96
|
+
if use_vlm:
|
|
97
|
+
from mlx_vlm import stream_generate as vlm_stream
|
|
98
|
+
|
|
99
|
+
chunks = vlm_stream(model, tokenizer, prompt=prompt, image=image, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model, draft_kind="mtp")
|
|
100
|
+
else:
|
|
101
|
+
from mlx_lm import stream_generate as lm_stream
|
|
102
|
+
|
|
103
|
+
chunks = lm_stream(model, tokenizer, prompt=prompt, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model)
|
|
104
|
+
text = ""
|
|
105
|
+
for chunk in chunks:
|
|
106
|
+
piece = chunk.text if hasattr(chunk, "text") else (chunk[0] if isinstance(chunk, tuple) else str(chunk))
|
|
107
|
+
text += piece
|
|
108
|
+
if any(marker in text for marker in stops):
|
|
109
|
+
break
|
|
110
|
+
return apply_stop_strings(text, stops)
|
|
24
111
|
|
|
25
112
|
|
|
26
113
|
class _GenerationMixin(_Core):
|
|
27
114
|
"""The chat generation half of :class:`LLMRouter`."""
|
|
28
115
|
|
|
29
|
-
def _build_prompt(
|
|
30
|
-
context =
|
|
31
|
-
|
|
116
|
+
def _build_prompt(
|
|
117
|
+
self, message: str, context: Optional[str], tokenizer, cite_sources: bool = True
|
|
118
|
+
) -> str:
|
|
119
|
+
system = _system_for(context, cite_sources)
|
|
32
120
|
if hasattr(tokenizer, "apply_chat_template"):
|
|
33
121
|
try:
|
|
34
122
|
msgs = [{"role": "system", "content": system}, {"role": "user", "content": message}]
|
|
@@ -37,9 +125,8 @@ class _GenerationMixin(_Core):
|
|
|
37
125
|
quiet()
|
|
38
126
|
return f"<|im_start|>system\n{system}<|im_end|>\n<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
|
|
39
127
|
|
|
40
|
-
def _build_vlm_prompt(self, model, processor, message: str, context: Optional[str], num_images: int) -> str:
|
|
41
|
-
|
|
42
|
-
system = _compose_system(SYSTEM_PROMPT, context)
|
|
128
|
+
def _build_vlm_prompt(self, model, processor, message: str, context: Optional[str], num_images: int, cite_sources: bool = True) -> str:
|
|
129
|
+
system = _system_for(context, cite_sources)
|
|
43
130
|
try:
|
|
44
131
|
from mlx_vlm import apply_chat_template
|
|
45
132
|
|
|
@@ -55,7 +142,7 @@ class _GenerationMixin(_Core):
|
|
|
55
142
|
)
|
|
56
143
|
except Exception as e:
|
|
57
144
|
print(f"β οΈ VLM chat template fallback: {e}")
|
|
58
|
-
return self._build_prompt(message, context, processor)
|
|
145
|
+
return self._build_prompt(message, context, processor, cite_sources)
|
|
59
146
|
|
|
60
147
|
async def generate_as(
|
|
61
148
|
self,
|
|
@@ -65,12 +152,35 @@ class _GenerationMixin(_Core):
|
|
|
65
152
|
max_tokens: int = 4096,
|
|
66
153
|
temperature: float = 0.2,
|
|
67
154
|
image_data: Optional[str] = None,
|
|
155
|
+
stop: Optional[List[str]] = None,
|
|
156
|
+
prefix: Optional[str] = None,
|
|
157
|
+
cite_sources: bool = True,
|
|
68
158
|
) -> str:
|
|
69
|
-
"""Generate with a request-scoped model without changing the default.
|
|
159
|
+
"""Generate with a request-scoped model without changing the default.
|
|
160
|
+
|
|
161
|
+
``stop`` ends the reply at the first of the given strings. Local
|
|
162
|
+
generation switches to the streaming backend to do it, so the tokens
|
|
163
|
+
after the stop are never produced rather than merely trimmed; the cloud
|
|
164
|
+
path forwards the list to the provider, which stops server-side. Absent
|
|
165
|
+
or empty means what it always meant β generate to ``max_tokens``.
|
|
166
|
+
|
|
167
|
+
``prefix`` **starts** the reply (v12.0.0): the characters are put in the
|
|
168
|
+
model's mouth rather than requested of it, so a caller that needs the
|
|
169
|
+
answer to begin ``{"thoughts": "`` gets that by construction instead of
|
|
170
|
+
by asking nicely and repairing the result. Locally this is a real
|
|
171
|
+
prefill β the text is appended to the templated prompt after the
|
|
172
|
+
generation marker, so the model continues from mid-token rather than
|
|
173
|
+
starting a fresh turn. The returned text always begins with it; see
|
|
174
|
+
:func:`~latticeai.models.router.loading.apply_prefix` for how the three
|
|
175
|
+
backends are reconciled.
|
|
176
|
+
"""
|
|
70
177
|
_selected, cached = self._model_snapshot(model_id)
|
|
71
178
|
if cached is None:
|
|
72
179
|
return "No model."
|
|
73
|
-
return await self._generate_cached(
|
|
180
|
+
return await self._generate_cached(
|
|
181
|
+
cached, message, context, max_tokens, temperature, image_data, stop, prefix,
|
|
182
|
+
cite_sources,
|
|
183
|
+
)
|
|
74
184
|
|
|
75
185
|
async def generate(
|
|
76
186
|
self,
|
|
@@ -79,8 +189,14 @@ class _GenerationMixin(_Core):
|
|
|
79
189
|
max_tokens: int = 4096,
|
|
80
190
|
temperature: float = 0.2,
|
|
81
191
|
image_data: Optional[str] = None,
|
|
192
|
+
stop: Optional[List[str]] = None,
|
|
193
|
+
prefix: Optional[str] = None,
|
|
194
|
+
cite_sources: bool = True,
|
|
82
195
|
) -> str:
|
|
83
|
-
return await self.generate_as(
|
|
196
|
+
return await self.generate_as(
|
|
197
|
+
None, message, context, max_tokens, temperature, image_data, stop, prefix,
|
|
198
|
+
cite_sources,
|
|
199
|
+
)
|
|
84
200
|
|
|
85
201
|
async def _generate_cached(
|
|
86
202
|
self,
|
|
@@ -90,51 +206,81 @@ class _GenerationMixin(_Core):
|
|
|
90
206
|
max_tokens: int,
|
|
91
207
|
temperature: float,
|
|
92
208
|
image_data: Optional[str],
|
|
209
|
+
stop: Optional[List[str]] = None,
|
|
210
|
+
prefix: Optional[str] = None,
|
|
211
|
+
cite_sources: bool = True,
|
|
93
212
|
) -> str:
|
|
94
213
|
if isinstance(cached, CloudModel):
|
|
95
|
-
return await self._cloud_generate(
|
|
214
|
+
return await self._cloud_generate(
|
|
215
|
+
cached, message, context, max_tokens, temperature, stop, prefix, cite_sources
|
|
216
|
+
)
|
|
96
217
|
|
|
97
218
|
model, tokenizer, draft_model, loader_kind = self._unpack_local_cache(cached)
|
|
98
219
|
use_vlm = loader_kind == "mlx_vlm"
|
|
99
220
|
prompt = (
|
|
100
|
-
self._build_vlm_prompt(
|
|
221
|
+
self._build_vlm_prompt(
|
|
222
|
+
model, tokenizer, message, context, 1 if image_data else 0, cite_sources
|
|
223
|
+
)
|
|
101
224
|
if use_vlm
|
|
102
|
-
else self._build_prompt(message, context, tokenizer)
|
|
225
|
+
else self._build_prompt(message, context, tokenizer, cite_sources)
|
|
103
226
|
)
|
|
104
|
-
|
|
227
|
+
# The prefill. Appended *after* the chat template's generation marker,
|
|
228
|
+
# so the first sampled token continues these characters instead of
|
|
229
|
+
# opening a reply. Nothing else in the pipeline changes.
|
|
230
|
+
if prefix:
|
|
231
|
+
prompt = f"{prompt}{prefix}"
|
|
232
|
+
stops = [marker for marker in (stop or []) if marker]
|
|
233
|
+
|
|
105
234
|
loop = asyncio.get_event_loop()
|
|
106
|
-
|
|
235
|
+
|
|
107
236
|
def _gen():
|
|
108
237
|
import mlx.core as mx # type: ignore[no-redef]
|
|
109
238
|
|
|
110
239
|
mx.set_default_device(mx.gpu) # type: ignore[arg-type]
|
|
240
|
+
# Decoded here, not above: base64 + PIL is CPU work, and this
|
|
241
|
+
# function is the part that runs off the event loop.
|
|
242
|
+
image = self._prep_image(image_data) if image_data else None
|
|
243
|
+
sampler = _mlx_sampler(temperature)
|
|
244
|
+
if stops:
|
|
245
|
+
return _stream_until_stop(
|
|
246
|
+
model, tokenizer, prompt, image, max_tokens, sampler, draft_model,
|
|
247
|
+
use_vlm, stops,
|
|
248
|
+
)
|
|
111
249
|
if use_vlm:
|
|
112
250
|
from mlx_vlm import generate as vlm_gen
|
|
113
|
-
return vlm_gen(model, tokenizer, prompt=prompt, image=
|
|
251
|
+
return vlm_gen(model, tokenizer, prompt=prompt, image=image, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model, draft_kind="mtp")
|
|
114
252
|
from mlx_lm import generate as lm_gen
|
|
115
|
-
return lm_gen(model, tokenizer, prompt=prompt, max_tokens=max_tokens, sampler=
|
|
253
|
+
return lm_gen(model, tokenizer, prompt=prompt, max_tokens=max_tokens, sampler=sampler, draft_model=draft_model)
|
|
116
254
|
result = await loop.run_in_executor(executor, _gen)
|
|
117
255
|
# mlx-vlm might return a GenerationResult object; extract the text
|
|
118
|
-
if hasattr(result, "text")
|
|
119
|
-
|
|
120
|
-
return normalize_branding(str(result))
|
|
256
|
+
text = result.text if hasattr(result, "text") else str(result)
|
|
257
|
+
return apply_prefix(prefix, normalize_branding(apply_stop_strings(text, stops)))
|
|
121
258
|
|
|
122
|
-
async def _cloud_generate(self, cloud: CloudModel, message: str, context: Optional[str], max_tokens: int, temperature: float) -> str:
|
|
123
|
-
|
|
124
|
-
|
|
259
|
+
async def _cloud_generate(self, cloud: CloudModel, message: str, context: Optional[str], max_tokens: int, temperature: float, stop: Optional[List[str]] = None, prefix: Optional[str] = None, cite_sources: bool = True) -> str:
|
|
260
|
+
system = _system_for(context, cite_sources)
|
|
261
|
+
stops = [marker for marker in (stop or []) if marker]
|
|
262
|
+
extra = {"stop": stops} if stops else {}
|
|
263
|
+
messages = [
|
|
264
|
+
{"role": "system", "content": system},
|
|
265
|
+
{"role": "user", "content": message},
|
|
266
|
+
]
|
|
267
|
+
# Assistant prefill: the OpenAI-compatible way to say "continue this".
|
|
268
|
+
# Local servers (vLLM, llama.cpp, LM Studio, Ollama) honour it; a
|
|
269
|
+
# provider that does not simply reads it as context, and `apply_prefix`
|
|
270
|
+
# makes both answers the same shape for the caller.
|
|
271
|
+
if prefix:
|
|
272
|
+
messages.append({"role": "assistant", "content": prefix})
|
|
125
273
|
try:
|
|
126
274
|
response = await cloud.client.chat.completions.create(
|
|
127
275
|
model=cloud.model,
|
|
128
|
-
messages=
|
|
129
|
-
{"role": "system", "content": system},
|
|
130
|
-
{"role": "user", "content": message},
|
|
131
|
-
],
|
|
276
|
+
messages=messages,
|
|
132
277
|
max_tokens=max_tokens,
|
|
133
278
|
temperature=temperature,
|
|
279
|
+
**extra,
|
|
134
280
|
)
|
|
135
281
|
except Exception as e:
|
|
136
282
|
raise RuntimeError(self._local_server_error_hint(cloud, e)) from e
|
|
137
|
-
return normalize_branding(response.choices[0].message.content or "")
|
|
283
|
+
return apply_prefix(prefix, normalize_branding(response.choices[0].message.content or ""))
|
|
138
284
|
|
|
139
285
|
async def stream_generate_as(
|
|
140
286
|
self,
|
|
@@ -25,6 +25,7 @@ import gc
|
|
|
25
25
|
from concurrent.futures import ThreadPoolExecutor
|
|
26
26
|
from typing import Any, Dict, List, Optional
|
|
27
27
|
|
|
28
|
+
from latticeai.core.quiet import quiet
|
|
28
29
|
from latticeai.models.model_providers import (
|
|
29
30
|
OPENAI_COMPATIBLE_PROVIDERS,
|
|
30
31
|
PROVIDER_MODEL_CATALOG,
|
|
@@ -120,12 +121,145 @@ def ensure_mlx_runtime() -> None:
|
|
|
120
121
|
def _mlx_sampler(temperature: float):
|
|
121
122
|
"""Build an MLX sampler callable for the given temperature.
|
|
122
123
|
|
|
123
|
-
|
|
124
|
-
MLX
|
|
125
|
-
|
|
124
|
+
Until v11.9.0 this discarded its argument and returned ``None``, which let
|
|
125
|
+
MLX pick its own default sampler. The cost was invisible and large: the
|
|
126
|
+
agent loop asks for 0.1 when it wants a tool call and 0.0 on the strict
|
|
127
|
+
verification re-ask, and **locally neither did anything**. A cloud model got
|
|
128
|
+
the deterministic re-ask it was designed around; the 2B model on this
|
|
129
|
+
machine got the same creative sampler it had on the first attempt, and the
|
|
130
|
+
"one strict retry" was one more roll of the same dice.
|
|
131
|
+
|
|
132
|
+
``mlx_lm.sample_utils.make_sampler`` is the sampler both backends accept β
|
|
133
|
+
``mlx_vlm`` takes the callable straight through β and it is already an
|
|
134
|
+
installed dependency of the text path. If it cannot be imported (an
|
|
135
|
+
MLX-VLM-only install, a version that moved it) the answer is the old one:
|
|
136
|
+
``None``, the backend's default. A missing sampler must never be a failed
|
|
137
|
+
generation.
|
|
126
138
|
"""
|
|
127
|
-
|
|
128
|
-
|
|
139
|
+
try:
|
|
140
|
+
from mlx_lm.sample_utils import make_sampler
|
|
141
|
+
except Exception: # noqa: BLE001 β an absent sampler is not a failure
|
|
142
|
+
quiet("MLX sampler unavailable; using the backend default")
|
|
143
|
+
return None
|
|
144
|
+
try:
|
|
145
|
+
return make_sampler(temp=float(temperature))
|
|
146
|
+
except Exception: # noqa: BLE001 β a signature change must not stop a run
|
|
147
|
+
quiet("MLX sampler construction failed; using the backend default")
|
|
148
|
+
return None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _declares_vision(model_id: str) -> bool:
|
|
152
|
+
"""Whether this checkpoint's own config says it is multimodal.
|
|
153
|
+
|
|
154
|
+
Read from ``config.json`` rather than decided from the model's *name*: a
|
|
155
|
+
list of known vision families is a list that is wrong about every model
|
|
156
|
+
published after it was written, and "which models work" is exactly the
|
|
157
|
+
question this product must not answer with a hardcoded roster.
|
|
158
|
+
|
|
159
|
+
The three markers are the ones every MLX-VLM architecture carries β a
|
|
160
|
+
nested ``vision_config``, a ``vision_tower``, or an ``image_token_index``
|
|
161
|
+
for the placeholder the processor substitutes. A config that carries none
|
|
162
|
+
of them describes a text model.
|
|
163
|
+
"""
|
|
164
|
+
import json as _json
|
|
165
|
+
from pathlib import Path as _Path
|
|
166
|
+
|
|
167
|
+
from .local_models import hf_model_dir
|
|
168
|
+
|
|
169
|
+
raw = str(model_id or "").strip()
|
|
170
|
+
candidates = []
|
|
171
|
+
explicit = _Path(raw).expanduser()
|
|
172
|
+
if raw and explicit.exists():
|
|
173
|
+
candidates.append(explicit / "config.json")
|
|
174
|
+
try:
|
|
175
|
+
candidates.append(hf_model_dir(raw) / "config.json")
|
|
176
|
+
except Exception as error: # noqa: BLE001 β an unresolvable id is not a claim
|
|
177
|
+
quiet(f"vision probe skipped for {raw}: {error}")
|
|
178
|
+
for config_path in candidates:
|
|
179
|
+
try:
|
|
180
|
+
if not config_path.exists():
|
|
181
|
+
continue
|
|
182
|
+
data = _json.loads(config_path.read_text(encoding="utf-8"))
|
|
183
|
+
except Exception as error: # noqa: BLE001 β an unreadable config is not a claim
|
|
184
|
+
quiet(f"vision probe skipped for {config_path}: {error}")
|
|
185
|
+
continue
|
|
186
|
+
if any(key in data for key in ("vision_config", "vision_tower", "image_token_index")):
|
|
187
|
+
return True
|
|
188
|
+
return False
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _text_fallback_allowed(
|
|
192
|
+
model_id: str, target_model_id: str, model_type: Optional[str]
|
|
193
|
+
) -> bool:
|
|
194
|
+
"""Whether a failed multimodal load may be retried as a text load.
|
|
195
|
+
|
|
196
|
+
Every local model is routed to MLX-VLM first, because the recommended
|
|
197
|
+
defaults are multimodal. Until v12.0.0 the retry on the text path was
|
|
198
|
+
gated on the model *id* matching Gemma 4 β so a plain text model MLX-VLM
|
|
199
|
+
cannot open (a 0.5B Qwen, an AWQ checkpoint, any fine-tune with a text-only
|
|
200
|
+
architecture) failed to load at all, with MLX-LM sitting right there able
|
|
201
|
+
to open it. That is a roster deciding what runs, and the roster was one
|
|
202
|
+
family long. "Every small model works" cannot be true while loading is
|
|
203
|
+
gated on a name.
|
|
204
|
+
|
|
205
|
+
So the rule is now two rules, and the second is **additive**:
|
|
206
|
+
|
|
207
|
+
* the v11.9.0 Gemma 4 rule, unchanged β a Gemma 4 that is not the unified
|
|
208
|
+
(vision) architecture may always retry as text;
|
|
209
|
+
* and, new, **any model whose own ``config.json`` does not declare
|
|
210
|
+
vision.** Read from the checkpoint rather than decided from the name, so
|
|
211
|
+
a model published tomorrow is judged the same way as one published last
|
|
212
|
+
year.
|
|
213
|
+
|
|
214
|
+
A config that *does* declare vision is never silently downgraded: answering
|
|
215
|
+
an image request from a text-only load is worse than refusing it.
|
|
216
|
+
"""
|
|
217
|
+
if lm_load is None:
|
|
218
|
+
return False
|
|
219
|
+
if (model_type or "").strip().lower() == "gemma4_unified":
|
|
220
|
+
return False
|
|
221
|
+
if _is_gemma4_model_id(model_id):
|
|
222
|
+
return True
|
|
223
|
+
return not _declares_vision(target_model_id)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def apply_prefix(prefix: Optional[str], text: str) -> str:
|
|
227
|
+
"""Guarantee ``text`` begins with ``prefix``, without doubling it.
|
|
228
|
+
|
|
229
|
+
The seam's contract for a forced prefix (v12.0.0) is one sentence: *the
|
|
230
|
+
reply starts with these characters*. How it got there differs by backend β
|
|
231
|
+
the local path really does prefill the prompt with them, so the model never
|
|
232
|
+
generated them and they have to be put back; a cloud provider handed an
|
|
233
|
+
assistant prefill message may echo them, may not, and may ignore the
|
|
234
|
+
message entirely. One normalisation covers all three, so the caller parses
|
|
235
|
+
one shape whichever answered.
|
|
236
|
+
|
|
237
|
+
Leading whitespace is dropped when the reply already carries the prefix:
|
|
238
|
+
a chat template that emits ``"\\n"`` before the continuation would otherwise
|
|
239
|
+
make ``startswith`` false for a reply that plainly does start with it.
|
|
240
|
+
"""
|
|
241
|
+
if not prefix:
|
|
242
|
+
return text
|
|
243
|
+
stripped = text.lstrip()
|
|
244
|
+
if stripped.startswith(prefix):
|
|
245
|
+
return stripped
|
|
246
|
+
return f"{prefix}{text}"
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def apply_stop_strings(text: str, stop: Optional[List[str]]) -> str:
|
|
250
|
+
"""Cut ``text`` at the earliest stop string, if any is present.
|
|
251
|
+
|
|
252
|
+
Shared by the buffered and streaming local paths so "where does this reply
|
|
253
|
+
end" has one answer. Empty stop strings are ignored: a caller that sends
|
|
254
|
+
``""`` means "no stop", not "stop before the first character".
|
|
255
|
+
"""
|
|
256
|
+
if not stop:
|
|
257
|
+
return text
|
|
258
|
+
cut = min(
|
|
259
|
+
(text.index(marker) for marker in stop if marker and marker in text),
|
|
260
|
+
default=-1,
|
|
261
|
+
)
|
|
262
|
+
return text if cut < 0 else text[:cut]
|
|
129
263
|
|
|
130
264
|
|
|
131
265
|
class _LoadingMixin(_Core):
|
|
@@ -170,7 +304,6 @@ class _LoadingMixin(_Core):
|
|
|
170
304
|
|
|
171
305
|
def _load():
|
|
172
306
|
mx.set_default_device(mx.gpu)
|
|
173
|
-
is_gemma4 = _is_gemma4_model_id(model_id)
|
|
174
307
|
model_type = _local_model_type(target_model_id) or _local_model_type(model_id)
|
|
175
308
|
loader_kind = "mlx_vlm"
|
|
176
309
|
|
|
@@ -180,11 +313,19 @@ class _LoadingMixin(_Core):
|
|
|
180
313
|
print(f"π Loading Target (VLM Mode): {target_model_id}...")
|
|
181
314
|
model, tokenizer = vlm_load(target_model_id)
|
|
182
315
|
except Exception as vlm_error:
|
|
183
|
-
if not (
|
|
316
|
+
if not _text_fallback_allowed(model_id, target_model_id, model_type):
|
|
184
317
|
raise
|
|
185
|
-
print(f"β οΈ
|
|
318
|
+
print(f"β οΈ MLX-VLM load failed; retrying the MLX-LM text path: {vlm_error}")
|
|
186
319
|
print(f"π Loading Target (LM Mode): {target_model_id}...")
|
|
187
|
-
|
|
320
|
+
try:
|
|
321
|
+
model, tokenizer = lm_load(target_model_id)
|
|
322
|
+
except Exception:
|
|
323
|
+
# The text path is a *fallback*: when it fails too, the
|
|
324
|
+
# useful diagnosis is the first failure, from the loader
|
|
325
|
+
# this model was routed to. Raising the second one would
|
|
326
|
+
# tell an operator their multimodal model is not a text
|
|
327
|
+
# model, which they knew.
|
|
328
|
+
raise vlm_error from None
|
|
188
329
|
loader_kind = "mlx_lm"
|
|
189
330
|
|
|
190
331
|
draft_model = None
|
|
@@ -45,8 +45,10 @@ def build_access_runtime(
|
|
|
45
45
|
# extend this trust to public or non-loopback bindings, even if an invalid
|
|
46
46
|
# caller constructs this runtime with ``require_auth=False`` directly.
|
|
47
47
|
externally_reachable = is_externally_reachable(config)
|
|
48
|
+
# One flag, not two: "authentication is required" is exactly
|
|
49
|
+
# ``not trusted_local_owner``, so deriving it separately only creates a
|
|
50
|
+
# pair that a later edit can put out of step.
|
|
48
51
|
trusted_local_owner = not require_auth and not externally_reachable
|
|
49
|
-
effective_require_auth = bool(require_auth or externally_reachable)
|
|
50
52
|
|
|
51
53
|
def get_user_role(email: str, users: Optional[Dict] = None) -> str:
|
|
52
54
|
users = users or load_users()
|
|
@@ -128,9 +130,10 @@ def build_access_runtime(
|
|
|
128
130
|
# shared local vaults, and Local User profile behavior compatible;
|
|
129
131
|
# get_user_role() supplies the explicit owner authorization role.
|
|
130
132
|
return ""
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
133
|
+
# Reaching here means the caller is not the trusted local owner and no
|
|
134
|
+
# session produced an email β i.e. authentication is required and was
|
|
135
|
+
# not supplied. There is no third outcome to fall through to.
|
|
136
|
+
raise http_exception(status_code=401, detail="μΈμ¦μ΄ νμν©λλ€.")
|
|
134
137
|
|
|
135
138
|
def require_admin(request: request_type) -> tuple[str, Dict]:
|
|
136
139
|
users = load_users()
|
|
@@ -13,7 +13,16 @@ was requested.
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
-
from typing import Any, Callable, Mapping
|
|
16
|
+
from typing import Any, Callable, Mapping, Optional, Tuple
|
|
17
|
+
|
|
18
|
+
from latticeai.core.embedding_providers.autodetect import (
|
|
19
|
+
AUTO_PROVIDER,
|
|
20
|
+
Detection,
|
|
21
|
+
detect_embedder,
|
|
22
|
+
resolve_auto_provider,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
_FALLBACK_NAMES = {"", "hash", "local", "fallback"}
|
|
17
26
|
|
|
18
27
|
|
|
19
28
|
def build_embedder_runtime(
|
|
@@ -21,16 +30,26 @@ def build_embedder_runtime(
|
|
|
21
30
|
config: Any,
|
|
22
31
|
profile: Mapping[str, Any],
|
|
23
32
|
resolve_embedder: Callable[..., Any],
|
|
33
|
+
detect: Optional[Callable[..., Detection]] = None,
|
|
24
34
|
) -> Any:
|
|
25
|
-
"""Resolve the configured embedding provider once.
|
|
35
|
+
"""Resolve the configured embedding provider once.
|
|
26
36
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
37
|
+
The resolution order is the product's, unchanged, with one addition at the
|
|
38
|
+
end: detection. What is on this machine is *always* looked up (so
|
|
39
|
+
``GET /api/embeddings/status`` can say "a real embedder is right there"),
|
|
40
|
+
and it changes the resolution only when the operator asked for it with
|
|
41
|
+
``LATTICEAI_EMBEDDING_PROVIDER=auto``. The finding rides back on the
|
|
42
|
+
resolved object as ``detected`` β an added attribute, so every existing
|
|
43
|
+
reader of ``ResolvedEmbedder`` is untouched.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
provider, model, dim = _configured(config, profile)
|
|
47
|
+
detection = (detect or detect_embedder)(
|
|
48
|
+
configured_provider=provider, configured_model=model
|
|
49
|
+
)
|
|
50
|
+
provider, model, dim = resolve_auto_provider(provider, model, dim, detection)
|
|
32
51
|
|
|
33
|
-
|
|
52
|
+
resolved = resolve_embedder(
|
|
34
53
|
provider,
|
|
35
54
|
model=model,
|
|
36
55
|
base_url=config.embedding_base_url,
|
|
@@ -38,8 +57,23 @@ def build_embedder_runtime(
|
|
|
38
57
|
dim=dim,
|
|
39
58
|
timeout=config.embedding_timeout,
|
|
40
59
|
extra={"target": config.embedding_custom_target},
|
|
41
|
-
probe=provider not in
|
|
60
|
+
probe=provider not in _FALLBACK_NAMES,
|
|
42
61
|
)
|
|
62
|
+
try:
|
|
63
|
+
resolved.detected = detection
|
|
64
|
+
except AttributeError: # pragma: no cover - a stand-in without __dict__
|
|
65
|
+
pass
|
|
66
|
+
return resolved
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _configured(config: Any, profile: Mapping[str, Any]) -> Tuple[str, str, int]:
|
|
70
|
+
"""``(provider, model, dim)`` from configuration and the named profile."""
|
|
71
|
+
provider = config.embedding_provider
|
|
72
|
+
model = config.embedding_model or str(profile.get("model") or "")
|
|
73
|
+
dim = config.embedding_dim or int(profile.get("dimensions") or 0)
|
|
74
|
+
if config.embedding_profile and provider in _FALLBACK_NAMES | {AUTO_PROVIDER}:
|
|
75
|
+
provider = str(profile.get("provider") or provider)
|
|
76
|
+
return provider, model, dim
|
|
43
77
|
|
|
44
78
|
|
|
45
79
|
__all__ = ["build_embedder_runtime"]
|