ltcai 11.0.1 → 11.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -43
- package/docs/CHANGELOG.md +61 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/FEATURE_AUDIT_v11.2.0.md +393 -0
- package/docs/LAYOUT_REBUILD_SPEC.md +9 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +1 -1
- package/docs/PERFORMANCE.md +71 -18
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/architecture.md +6 -2
- package/docs/kg-schema.md +1 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/embeddings.py +12 -37
- package/lattice_brain/gates.py +125 -0
- package/lattice_brain/graph/discovery_index.py +30 -32
- package/lattice_brain/graph/fusion.py +35 -4
- package/lattice_brain/graph/image_vectors.py +230 -0
- package/lattice_brain/graph/ingest.py +11 -5
- package/lattice_brain/graph/projection.py +66 -8
- package/lattice_brain/graph/provenance.py +27 -2
- package/lattice_brain/graph/retrieval.py +113 -2
- package/lattice_brain/graph/retrieval_docgen.py +6 -6
- package/lattice_brain/graph/schema.py +18 -0
- package/lattice_brain/graph/store.py +9 -0
- package/lattice_brain/graph/vector_index/selector.py +32 -2
- package/lattice_brain/ingestion.py +363 -10
- package/lattice_brain/multimodal.py +1258 -0
- package/lattice_brain/portability.py +169 -32
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/lattice_brain/sealed_box.py +244 -0
- package/lattice_brain/self_model.py +77 -22
- package/lattice_brain/synthesis.py +24 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/brain_intelligence.py +4 -0
- package/latticeai/api/chat.py +11 -0
- package/latticeai/api/chat_helpers.py +16 -3
- package/latticeai/api/chat_hybrid.py +32 -1
- package/latticeai/api/features.py +70 -0
- package/latticeai/api/local_files.py +102 -0
- package/latticeai/api/memory.py +128 -1
- package/latticeai/api/portability.py +39 -4
- package/latticeai/api/review_queue.py +126 -0
- package/latticeai/api/search.py +16 -2
- package/latticeai/core/agent.py +59 -2
- package/latticeai/core/agent_prompts.py +66 -0
- package/latticeai/core/config.py +4 -1
- package/latticeai/core/context_builder.py +98 -11
- package/latticeai/core/embedding_providers.py +528 -0
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/messages.py +180 -0
- package/latticeai/core/model_compat.py +73 -2
- package/latticeai/core/workspace_os.py +43 -0
- package/latticeai/core/workspace_os_constants.py +1 -1
- package/latticeai/core/workspace_reorganization.py +335 -0
- package/latticeai/models/model_providers.py +12 -4
- package/latticeai/runtime/build_phases.py +33 -2
- package/latticeai/runtime/chat_wiring.py +4 -0
- package/latticeai/runtime/feature_toggle_wiring.py +163 -0
- package/latticeai/runtime/persistence_runtime.py +41 -4
- package/latticeai/runtime/router_registration.py +11 -0
- package/latticeai/runtime/runtime_context.py +1 -0
- package/latticeai/services/app_context.py +8 -0
- package/latticeai/services/architecture_readiness.py +1 -1
- package/latticeai/services/automation_intelligence.py +22 -2
- package/latticeai/services/brain_intelligence.py +123 -7
- package/latticeai/services/change_proposals.py +50 -10
- package/latticeai/services/command_center.py +10 -4
- package/latticeai/services/feature_toggles.py +502 -0
- package/latticeai/services/folder_watch.py +122 -1
- package/latticeai/services/hybrid_chat.py +56 -5
- package/latticeai/services/interop_bridges.py +978 -0
- package/latticeai/services/memory_service.py +34 -0
- package/latticeai/services/model_capability_registry.py +434 -261
- package/latticeai/services/model_catalog.py +95 -61
- package/latticeai/services/model_recommendation.py +18 -11
- package/latticeai/services/model_runtime.py +1 -1
- package/latticeai/services/multimodal_ports.py +112 -0
- package/latticeai/services/obsidian_bridge.py +16 -25
- package/latticeai/services/product_readiness.py +1 -1
- package/latticeai/services/search_service.py +149 -2
- package/latticeai/services/self_model_service.py +171 -0
- package/latticeai/services/tool_dispatch.py +4 -0
- package/latticeai/services/voice_capture.py +27 -1
- package/latticeai/setup/auto_setup.py +27 -30
- package/latticeai/setup/wizard.py +77 -44
- package/package.json +1 -1
- package/scripts/check_current_release_docs.mjs +1 -1
- package/scripts/check_server_i18n.mjs +1 -0
- package/scripts/release_screen_claims.json +22 -0
- package/scripts/verify_hf_model_registry.py +253 -218
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +37 -37
- package/static/app/assets/{Act-D4zSxFR-.js → Act-AWf0SAKp.js} +1 -1
- package/static/app/assets/{AdminConsole-w5jBfPt2.js → AdminConsole-D0u8Tiyj.js} +1 -1
- package/static/app/assets/{Brain-C2EqQg74.js → Brain-tuhI4sOC.js} +1 -1
- package/static/app/assets/BrainHome-Ts7G_Ila.js +2 -0
- package/static/app/assets/BrainSignals-jMYgQ2Ar.js +1 -0
- package/static/app/assets/{Capture-DPqpGK8d.js → Capture-CqOSzyPr.js} +1 -1
- package/static/app/assets/{CommandPalette-CNf7h5fp.js → CommandPalette-DC0Bzh-I.js} +1 -1
- package/static/app/assets/{Library-BN0HYOfc.js → Library-CX-bbhmK.js} +1 -1
- package/static/app/assets/{LivingBrain-Dfq_wEDI.js → LivingBrain-DBwhto14.js} +1 -1
- package/static/app/assets/{ProductFlow-B-3O0rNV.js → ProductFlow-BHA2cfKI.js} +1 -1
- package/static/app/assets/{ReviewCard-gZ-tdqFM.js → ReviewCard-BUhCKRNM.js} +1 -1
- package/static/app/assets/{System-BElUcSSw.js → System-Bu2t5hn1.js} +1 -1
- package/static/app/assets/arrow-left-Dzwa5zRb.js +1 -0
- package/static/app/assets/{bot--qYHMtkP.js → bot-Cia42c2h.js} +1 -1
- package/static/app/assets/brain-DJMoqrwx.js +1 -0
- package/static/app/assets/{button-51Z3rsuv.js → button-2j2Ijzgq.js} +1 -1
- package/static/app/assets/{circle-pause-CMIiMaQl.js → circle-pause-BEFeWpVW.js} +1 -1
- package/static/app/assets/{circle-play-DZoO_cfG.js → circle-play-ujXMcHxl.js} +1 -1
- package/static/app/assets/{cpu-Bs6uc9W9.js → cpu-k4awryFq.js} +1 -1
- package/static/app/assets/{download-G-2olkWz.js → download-DFbLJ_ig.js} +1 -1
- package/static/app/assets/{folder-open-CTOspnmb.js → folder-open-7y_b6xkM.js} +1 -1
- package/static/app/assets/{hard-drive-CewHWJhn.js → hard-drive-Bidh02Kr.js} +1 -1
- package/static/app/assets/{index-D7Rr-J2Y.js → index-BpYkzcVm.js} +3 -3
- package/static/app/assets/index-DwDl9-8Y.css +2 -0
- package/static/app/assets/{input-D4w_BZWl.js → input-DSlJJxRs.js} +1 -1
- package/static/app/assets/{permissionCopy-CosBEXAZ.js → permissionCopy-Bpb83Hx9.js} +1 -1
- package/static/app/assets/{primitives-d0g9pvzS.js → primitives-BCx6TvfG.js} +1 -1
- package/static/app/assets/search-Cgy8cCFJ.js +1 -0
- package/static/app/assets/{share-2-NmD7e_oV.js → share-2-BH1M-WNi.js} +1 -1
- package/static/app/assets/{shield-alert-CcQeMuju.js → shield-alert-BlKdBXcG.js} +1 -1
- package/static/app/assets/{textarea-BPAJDc-0.js → textarea-CCWbUfFB.js} +1 -1
- package/static/app/assets/{useFocusTrap-C7YLdTBC.js → useFocusTrap-YdHQ7pJ1.js} +1 -1
- package/static/app/assets/{useQuery-DRyD9opW.js → useQuery-CXQiwbVT.js} +1 -1
- package/static/app/assets/{utils-DG1_ExrP.js → utils-zqPZJxdx.js} +2 -2
- package/static/app/assets/{workspace-CWVf3gsI.js → workspace-DXTihhfU.js} +1 -1
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/static/app/assets/BrainHome-CvXS6XiQ.js +0 -2
- package/static/app/assets/BrainSignals-DOE_KhOU.js +0 -1
- package/static/app/assets/arrow-left-CFNIMjhv.js +0 -1
- package/static/app/assets/brain-DDCLjRqO.js +0 -1
- package/static/app/assets/index-CkzokZAj.css +0 -2
- package/static/app/assets/search-BLCYt75v.js +0 -1
|
@@ -1,25 +1,48 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
python3 scripts/verify_hf_model_registry.py
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
2
|
+
"""Verify the Lattice AI model capability registry against the Hugging Face API.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
python3 scripts/verify_hf_model_registry.py
|
|
6
|
+
python3 scripts/verify_hf_model_registry.py --out verification_report.json
|
|
7
|
+
|
|
8
|
+
What it does
|
|
9
|
+
------------
|
|
10
|
+
For every entry in the registry (recommended *and* recognised-only) it asks the
|
|
11
|
+
public HF REST API for the repo metadata and its file tree, then records:
|
|
12
|
+
|
|
13
|
+
* whether the repo exists and is reachable **without credentials**
|
|
14
|
+
* the Hub's canonical id, so a case drift in our catalog is caught
|
|
15
|
+
(``mlx-community/gemma-4-12b-it-4bit`` answers as ``…-12B-it-4bit``)
|
|
16
|
+
* the ``gated`` flag — a gated repo cannot be downloaded by our users
|
|
17
|
+
* ``library_name`` / tags, the config ``model_type``, downloads, likes,
|
|
18
|
+
``lastModified``
|
|
19
|
+
* the sibling files: ``config.json``, at least one ``.safetensors`` shard, and a
|
|
20
|
+
tokenizer file — plus the exact byte sum of every sibling, which is compared
|
|
21
|
+
against the ``size`` / ``download_size_gb`` recorded in the registry.
|
|
22
|
+
|
|
23
|
+
**It never downloads weights and never loads a model.** There is deliberately no
|
|
24
|
+
flag that could: the only network calls are two JSON GETs per repo. Verifying a
|
|
25
|
+
catalog must not cost the person running it a 20GB download.
|
|
26
|
+
|
|
27
|
+
The loadability verdict — and what it is *not*
|
|
28
|
+
----------------------------------------------
|
|
29
|
+
"Can this model actually load?" is answered **statically**, from three signals:
|
|
30
|
+
|
|
31
|
+
(a) the repo declares the MLX library (``library_name == "mlx"`` or an ``mlx``
|
|
32
|
+
tag), so an MLX-format conversion exists;
|
|
33
|
+
(b) the config architecture (``model_type``) is in SUPPORTED_MLX_ARCHITECTURES
|
|
34
|
+
below, i.e. a loader for it shipped in mlx-lm / mlx-vlm; and
|
|
35
|
+
(c) the community has downloaded it — a repo nobody has ever pulled is not
|
|
36
|
+
evidence of anything.
|
|
37
|
+
|
|
38
|
+
**This is not a load test.** It cannot detect a corrupt shard, a quantisation
|
|
39
|
+
the installed mlx build rejects, a tokenizer mismatch, or an mlx-vlm version
|
|
40
|
+
older than the architecture. A ``loadable`` verdict here means "nothing in the
|
|
41
|
+
published metadata says this cannot load", not "this loaded". The only authority
|
|
42
|
+
on a real load remains the loader plus the smoke test on the user's own machine.
|
|
43
|
+
|
|
44
|
+
Exit code: 0 when every recommended entry is reachable, ungated, correctly cased
|
|
45
|
+
and statically loadable; 1 otherwise.
|
|
23
46
|
"""
|
|
24
47
|
|
|
25
48
|
from __future__ import annotations
|
|
@@ -27,12 +50,11 @@ from __future__ import annotations
|
|
|
27
50
|
import argparse
|
|
28
51
|
import json
|
|
29
52
|
import sys
|
|
30
|
-
import time
|
|
31
53
|
import urllib.error
|
|
32
54
|
import urllib.request
|
|
33
55
|
from datetime import datetime, timezone
|
|
34
56
|
from pathlib import Path
|
|
35
|
-
from typing import Any, Dict, List, Optional
|
|
57
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
36
58
|
|
|
37
59
|
# Add repo root so we can import the registry directly
|
|
38
60
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
@@ -40,264 +62,277 @@ sys.path.insert(0, str(REPO_ROOT))
|
|
|
40
62
|
|
|
41
63
|
try:
|
|
42
64
|
from latticeai.services.model_capability_registry import (
|
|
65
|
+
RECOMMENDED,
|
|
43
66
|
ModelCapability,
|
|
44
67
|
get_all_capabilities,
|
|
45
68
|
)
|
|
46
|
-
except Exception as e:
|
|
69
|
+
except Exception as e: # pragma: no cover - import guard for standalone runs
|
|
47
70
|
print("ERROR: Could not import model_capability_registry:", e)
|
|
48
71
|
sys.exit(2)
|
|
49
72
|
|
|
50
73
|
|
|
51
74
|
HF_API = "https://huggingface.co/api/models/{repo}"
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
75
|
+
HF_TREE = "https://huggingface.co/api/models/{repo}/tree/main?recursive=true"
|
|
76
|
+
|
|
77
|
+
# Architectures with a loader in the MLX stack, as published by the projects
|
|
78
|
+
# themselves. Source (checked 2026-08-10):
|
|
79
|
+
# mlx-lm — https://github.com/ml-explore/mlx-lm/tree/main/mlx_lm/models
|
|
80
|
+
# mlx-vlm — https://github.com/Blaizzy/mlx-vlm/tree/main/mlx_vlm/models
|
|
81
|
+
# The module basename in those directories *is* the HF config ``model_type``,
|
|
82
|
+
# which is why this can be matched exactly rather than guessed. Entries here are
|
|
83
|
+
# limited to the architectures this registry actually ships or recognises; it is
|
|
84
|
+
# not a mirror of the full upstream list.
|
|
85
|
+
SUPPORTED_MLX_ARCHITECTURES: Dict[str, str] = {
|
|
86
|
+
# vision-language loaders (mlx-vlm)
|
|
87
|
+
"gemma4": "mlx-vlm",
|
|
88
|
+
"gemma4_unified": "mlx-vlm",
|
|
89
|
+
"gemma3": "mlx-vlm",
|
|
90
|
+
"qwen3_5": "mlx-vlm",
|
|
91
|
+
"qwen3_5_moe": "mlx-vlm",
|
|
92
|
+
"qwen3_vl": "mlx-vlm",
|
|
93
|
+
"qwen3_vl_moe": "mlx-vlm",
|
|
94
|
+
"qwen2_5_vl": "mlx-vlm",
|
|
95
|
+
"mllama": "mlx-vlm",
|
|
96
|
+
"llama4": "mlx-vlm",
|
|
97
|
+
# text loaders (mlx-lm)
|
|
98
|
+
"gpt_oss": "mlx-lm",
|
|
99
|
+
"lfm2": "mlx-lm",
|
|
100
|
+
"llama": "mlx-lm",
|
|
101
|
+
"qwen3": "mlx-lm",
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
#: Below this many all-time downloads we refuse to call an entry proven by the
|
|
105
|
+
#: community. It is a weak signal on purpose — it only ever *downgrades* a
|
|
106
|
+
#: verdict, it never promotes one.
|
|
107
|
+
MIN_COMMUNITY_DOWNLOADS = 100
|
|
108
|
+
|
|
109
|
+
VERDICT_LOADABLE = "loadable_static"
|
|
110
|
+
VERDICT_NEEDS_REVIEW = "needs_review"
|
|
111
|
+
VERDICT_UNAVAILABLE = "unavailable"
|
|
112
|
+
|
|
113
|
+
LIMITATIONS = [
|
|
114
|
+
"Static verdict only: no weights were downloaded and no model was loaded.",
|
|
115
|
+
"A 'loadable_static' verdict means the published metadata contains nothing "
|
|
116
|
+
"that rules out a load — not that a load was observed.",
|
|
117
|
+
"It cannot see a corrupt shard, an incompatible quantisation, a tokenizer "
|
|
118
|
+
"mismatch, or an installed mlx-vlm older than the architecture.",
|
|
119
|
+
"Anonymous requests only: a repo that needs credentials is reported as "
|
|
120
|
+
"unavailable, because that is what it is for our users.",
|
|
121
|
+
"The loader plus the on-device smoke test remain the only authority on "
|
|
122
|
+
"whether a model really runs.",
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _http_get(url: str, timeout: float = 20.0) -> Tuple[Optional[Any], Optional[int]]:
|
|
127
|
+
"""GET a JSON document. Returns ``(payload, http_status)``; payload is None on failure."""
|
|
128
|
+
req = urllib.request.Request(url, headers={"User-Agent": "LatticeAI-model-registry-verifier/2.0"})
|
|
57
129
|
try:
|
|
58
130
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
59
131
|
raw = resp.read().decode("utf-8", errors="replace")
|
|
60
|
-
if
|
|
61
|
-
return {}
|
|
62
|
-
return json.loads(raw)
|
|
132
|
+
return (json.loads(raw) if raw.strip() else {}), int(resp.status)
|
|
63
133
|
except urllib.error.HTTPError as e:
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return None
|
|
134
|
+
# 404 = gone. 401 = the Hub's answer for "gone or private" to an
|
|
135
|
+
# anonymous client; either way our users cannot download it.
|
|
136
|
+
return None, int(e.code)
|
|
68
137
|
except Exception as e:
|
|
69
|
-
print(f"
|
|
70
|
-
return None
|
|
138
|
+
print(f" net error {url}: {type(e).__name__}")
|
|
139
|
+
return None, None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _mlx_signal(info: Dict[str, Any]) -> bool:
|
|
143
|
+
"""True when the repo advertises MLX-format weights."""
|
|
144
|
+
tags = [str(t).lower() for t in (info.get("tags") or [])]
|
|
145
|
+
return str(info.get("library_name") or "").lower() == "mlx" or "mlx" in tags
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _architecture(info: Dict[str, Any]) -> str:
|
|
149
|
+
config = info.get("config") or {}
|
|
150
|
+
return str(config.get("model_type") or "").strip().lower()
|
|
71
151
|
|
|
72
152
|
|
|
73
|
-
def
|
|
74
|
-
|
|
153
|
+
def _siblings(repo: str) -> Tuple[List[Dict[str, Any]], Optional[int]]:
|
|
154
|
+
tree, status = _http_get(HF_TREE.format(repo=repo))
|
|
155
|
+
return (tree if isinstance(tree, list) else []), status
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def verify_one(cap: ModelCapability) -> Dict[str, Any]:
|
|
159
|
+
"""Measure one registry entry against the Hub. Metadata requests only."""
|
|
75
160
|
repo = cap.hf_repo_id
|
|
76
161
|
result: Dict[str, Any] = {
|
|
77
162
|
"id": cap.id,
|
|
78
163
|
"hf_repo_id": repo,
|
|
164
|
+
"lifecycle": cap.lifecycle,
|
|
79
165
|
"family": cap.family,
|
|
80
|
-
"size": cap.size,
|
|
81
166
|
"modality": cap.modality,
|
|
167
|
+
"registry_size": cap.size,
|
|
168
|
+
"registry_architecture": cap.architecture,
|
|
169
|
+
"checked_at": datetime.now(timezone.utc).isoformat(),
|
|
82
170
|
"hf_exists": False,
|
|
171
|
+
"http_status": None,
|
|
172
|
+
"canonical_id": None,
|
|
173
|
+
"canonical_case_matches": None,
|
|
174
|
+
"gated": None,
|
|
175
|
+
"library_name": None,
|
|
176
|
+
"tags_sample": [],
|
|
83
177
|
"pipeline_tag": None,
|
|
178
|
+
"architecture": None,
|
|
179
|
+
"architecture_matches_registry": None,
|
|
180
|
+
"architecture_supported_by": None,
|
|
181
|
+
"downloads": None,
|
|
84
182
|
"likes": None,
|
|
85
183
|
"lastModified": None,
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
184
|
+
"has_config": False,
|
|
185
|
+
"has_tokenizer": False,
|
|
186
|
+
"safetensors_count": 0,
|
|
187
|
+
"measured_size_gb": None,
|
|
188
|
+
"registry_size_gb": cap.download_size_gb,
|
|
189
|
+
"size_delta_gb": None,
|
|
190
|
+
"mlx_signal": False,
|
|
191
|
+
"verdict": VERDICT_UNAVAILABLE,
|
|
192
|
+
"reasons": [],
|
|
91
193
|
"notes": "",
|
|
92
|
-
"checked_at": datetime.now(timezone.utc).isoformat(),
|
|
93
194
|
}
|
|
195
|
+
reasons: List[str] = result["reasons"]
|
|
94
196
|
|
|
95
|
-
info = _http_get(HF_API.format(repo=repo))
|
|
197
|
+
info, status = _http_get(HF_API.format(repo=repo))
|
|
198
|
+
result["http_status"] = status
|
|
96
199
|
if info is None:
|
|
97
|
-
|
|
200
|
+
reasons.append(f"repo not reachable anonymously (HTTP {status})")
|
|
201
|
+
result["notes"] = "404/401 on the HF API — cannot be downloaded without credentials."
|
|
98
202
|
return result
|
|
99
203
|
|
|
100
204
|
result["hf_exists"] = True
|
|
205
|
+
result["canonical_id"] = info.get("id")
|
|
206
|
+
result["canonical_case_matches"] = info.get("id") == repo
|
|
207
|
+
result["gated"] = info.get("gated")
|
|
208
|
+
result["library_name"] = info.get("library_name")
|
|
209
|
+
result["tags_sample"] = [str(t) for t in (info.get("tags") or [])][:8]
|
|
101
210
|
result["pipeline_tag"] = info.get("pipeline_tag")
|
|
211
|
+
result["downloads"] = info.get("downloads")
|
|
102
212
|
result["likes"] = info.get("likes")
|
|
103
213
|
result["lastModified"] = info.get("lastModified")
|
|
104
|
-
result["license"] = (info.get("author") or "") + " / " + str(info.get("license", info.get("tags", ["?"])[0] if info.get("tags") else "?"))
|
|
105
|
-
tags = info.get("tags") or []
|
|
106
|
-
result["tags_sample"] = tags[:6]
|
|
107
|
-
|
|
108
|
-
# Siblings via /tree (light, shows filenames + simple types; size omitted in some)
|
|
109
|
-
files = _http_get(HF_FILES.format(repo=repo)) or []
|
|
110
|
-
names = []
|
|
111
|
-
if isinstance(files, list):
|
|
112
|
-
for f in files:
|
|
113
|
-
if isinstance(f, dict):
|
|
114
|
-
n = str(f.get("path") or f.get("rfilename") or "").strip()
|
|
115
|
-
if n:
|
|
116
|
-
names.append(n.lower())
|
|
117
|
-
|
|
118
|
-
has_config = any("config.json" in n for n in names)
|
|
119
|
-
has_tok = any("tokenizer" in n or n.endswith(".model") for n in names)
|
|
120
|
-
has_weights = any(n.endswith((".safetensors", ".bin", ".gguf", ".pt")) for n in names)
|
|
121
|
-
|
|
122
|
-
result["has_config_hint"] = has_config
|
|
123
|
-
result["has_tokenizer_hint"] = has_tok
|
|
124
|
-
result["has_weights_hint"] = has_weights
|
|
125
|
-
|
|
126
|
-
if not has_config:
|
|
127
|
-
result["notes"] += "No config.json visible in tree. "
|
|
128
|
-
if not has_tok:
|
|
129
|
-
result["notes"] += "No obvious tokenizer file. "
|
|
130
|
-
if cap.hardware and cap.hardware.min_ram_gb and cap.hardware.min_ram_gb > 12:
|
|
131
|
-
result["notes"] += "LARGE_MODEL: local load practical only on high-RAM systems (32GB+ Apple Silicon or CUDA recommended). Expect long first download. "
|
|
132
|
-
|
|
133
|
-
return result
|
|
134
|
-
|
|
135
214
|
|
|
136
|
-
|
|
137
|
-
""
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
215
|
+
arch = _architecture(info)
|
|
216
|
+
result["architecture"] = arch or None
|
|
217
|
+
result["architecture_matches_registry"] = bool(arch) and arch == cap.architecture
|
|
218
|
+
result["architecture_supported_by"] = SUPPORTED_MLX_ARCHITECTURES.get(arch)
|
|
219
|
+
result["mlx_signal"] = _mlx_signal(info)
|
|
220
|
+
|
|
221
|
+
files, _tree_status = _siblings(repo)
|
|
222
|
+
names = [str(f.get("path") or "") for f in files if isinstance(f, dict)]
|
|
223
|
+
lowered = [n.lower() for n in names]
|
|
224
|
+
result["has_config"] = "config.json" in lowered
|
|
225
|
+
result["has_tokenizer"] = any("tokenizer" in n or n.endswith(".model") for n in lowered)
|
|
226
|
+
result["safetensors_count"] = sum(1 for n in lowered if n.endswith(".safetensors"))
|
|
227
|
+
total_bytes = sum(int(f.get("size") or 0) for f in files if isinstance(f, dict))
|
|
228
|
+
if total_bytes:
|
|
229
|
+
measured = round(total_bytes / 1e9, 2)
|
|
230
|
+
result["measured_size_gb"] = measured
|
|
231
|
+
if cap.download_size_gb is not None:
|
|
232
|
+
result["size_delta_gb"] = round(measured - cap.download_size_gb, 2)
|
|
233
|
+
|
|
234
|
+
# ── static verdict ────────────────────────────────────────────────────────
|
|
235
|
+
if result["gated"]:
|
|
236
|
+
reasons.append(f"gated={result['gated']} — needs Hub credentials")
|
|
237
|
+
if not result["canonical_case_matches"]:
|
|
238
|
+
reasons.append(f"case drift: registry {repo!r} vs canonical {result['canonical_id']!r}")
|
|
239
|
+
if not result["mlx_signal"]:
|
|
240
|
+
reasons.append("no mlx library_name or mlx tag")
|
|
241
|
+
if result["architecture_supported_by"] is None:
|
|
242
|
+
reasons.append(f"architecture {arch or '?'} is not in SUPPORTED_MLX_ARCHITECTURES")
|
|
243
|
+
if not result["architecture_matches_registry"]:
|
|
244
|
+
reasons.append(f"architecture {arch or '?'} != registry {cap.architecture or '?'}")
|
|
245
|
+
if not result["has_config"]:
|
|
246
|
+
reasons.append("no config.json in the file tree")
|
|
247
|
+
if not result["has_tokenizer"]:
|
|
248
|
+
reasons.append("no tokenizer file in the file tree")
|
|
249
|
+
if result["safetensors_count"] < 1:
|
|
250
|
+
reasons.append("no .safetensors shard in the file tree")
|
|
251
|
+
downloads = result["downloads"] or 0
|
|
252
|
+
if downloads < MIN_COMMUNITY_DOWNLOADS:
|
|
253
|
+
reasons.append(f"only {downloads} downloads — too few to count as community-proven")
|
|
254
|
+
if result["size_delta_gb"] is not None and abs(result["size_delta_gb"]) > 0.2:
|
|
255
|
+
reasons.append(
|
|
256
|
+
f"size drift: measured {result['measured_size_gb']}GB vs registry {cap.download_size_gb}GB"
|
|
156
257
|
)
|
|
157
|
-
p = Path(path)
|
|
158
|
-
cfg = (p / "config.json").exists()
|
|
159
|
-
tok = any((p / n).exists() for n in ("tokenizer.json", "tokenizer_config.json", "tokenizer.model"))
|
|
160
|
-
out.update({"deep_ok": True, "has_config": cfg, "has_tokenizer": tok, "used": "snapshot_download(restricted)"})
|
|
161
|
-
except Exception as e:
|
|
162
|
-
out["error"] = str(e)[:300]
|
|
163
|
-
return out
|
|
164
258
|
|
|
259
|
+
result["verdict"] = VERDICT_LOADABLE if not reasons else VERDICT_NEEDS_REVIEW
|
|
260
|
+
return result
|
|
165
261
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
out["library"] = "transformers"
|
|
174
|
-
cfg = AutoConfig.from_pretrained(repo, trust_remote_code=False)
|
|
175
|
-
tok = AutoTokenizer.from_pretrained(repo, trust_remote_code=False, use_fast=True)
|
|
176
|
-
out["load_test_attempted"] = True
|
|
177
|
-
out["load_ok"] = bool(cfg) and bool(tok)
|
|
178
|
-
out["model_type"] = getattr(cfg, "model_type", None)
|
|
179
|
-
return out
|
|
180
|
-
except Exception as e1:
|
|
181
|
-
out["error"] = f"transformers: {str(e1)[:200]}"
|
|
182
|
-
# Fallback: mlx_lm or mlx_vlm config only (very light)
|
|
183
|
-
try:
|
|
184
|
-
# mlx-lm has from_pretrained but we avoid full weight if possible; just check import path
|
|
185
|
-
import importlib
|
|
186
|
-
if importlib.util.find_spec("mlx_lm"):
|
|
187
|
-
out["library"] = "mlx_lm (config only probe)"
|
|
188
|
-
# We don't call full load here to stay true to "no blind huge weights"
|
|
189
|
-
out["load_test_attempted"] = True
|
|
190
|
-
out["load_ok"] = True # assume if importable the path exists; user will hit real load later
|
|
191
|
-
out["notes"] = "mlx path present; full local load tested at runtime only"
|
|
192
|
-
return out
|
|
193
|
-
except Exception:
|
|
194
|
-
pass
|
|
195
|
-
out["load_test_attempted"] = True
|
|
196
|
-
return out
|
|
262
|
+
|
|
263
|
+
def _print_row(r: Dict[str, Any]) -> None:
|
|
264
|
+
mark = {VERDICT_LOADABLE: "OK ", VERDICT_NEEDS_REVIEW: "?? ", VERDICT_UNAVAILABLE: "XX "}[r["verdict"]]
|
|
265
|
+
size = f"{r['measured_size_gb']}GB" if r["measured_size_gb"] else "-"
|
|
266
|
+
print(f"{mark}{r['id']:<46} {size:>9} {str(r['architecture'] or '-'):<16} {r['lifecycle']}")
|
|
267
|
+
for reason in r["reasons"]:
|
|
268
|
+
print(f" · {reason}")
|
|
197
269
|
|
|
198
270
|
|
|
199
271
|
def main() -> int:
|
|
200
|
-
parser = argparse.ArgumentParser()
|
|
201
|
-
parser.add_argument("--deep", action="store_true", help="Also fetch tiny config+tokenizer via hf_hub snapshot (restricted)")
|
|
202
|
-
parser.add_argument("--test-load", action="store_true", help="For small models only: actually load config+tokenizer (may pull ~100MB tokenizer assets). Skips >~8GB models.")
|
|
272
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
203
273
|
parser.add_argument("--out", default="verification_report.json", help="Report filename (written to cwd)")
|
|
204
274
|
args = parser.parse_args()
|
|
205
275
|
|
|
206
276
|
caps = get_all_capabilities()
|
|
207
|
-
print("Lattice AI
|
|
208
|
-
print(f"
|
|
209
|
-
print(
|
|
210
|
-
print("-" * 88)
|
|
277
|
+
print("Lattice AI model registry verifier — HF metadata only, no downloads, no loads")
|
|
278
|
+
print(f"Entries: {len(caps)} Time: {datetime.now(timezone.utc).isoformat()}")
|
|
279
|
+
print("-" * 96)
|
|
211
280
|
|
|
212
281
|
results: List[Dict[str, Any]] = []
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
for
|
|
220
|
-
|
|
221
|
-
deep = {}
|
|
222
|
-
load = {}
|
|
223
|
-
|
|
224
|
-
is_large = False
|
|
225
|
-
try:
|
|
226
|
-
sz = float("".join(ch for ch in cap.size if ch.isdigit() or ch == ".") or "0")
|
|
227
|
-
if "GB" in cap.size and sz > 12:
|
|
228
|
-
is_large = True
|
|
229
|
-
large_limited += 1
|
|
230
|
-
except Exception:
|
|
231
|
-
pass
|
|
232
|
-
|
|
233
|
-
if args.deep:
|
|
234
|
-
deep = try_deep_config(cap.hf_repo_id, tmp)
|
|
235
|
-
time.sleep(0.4)
|
|
236
|
-
|
|
237
|
-
do_load = args.test_load and not is_large and ("4B" in cap.name or "E2B" in cap.name or "2.7GB" in cap.size or "3.6GB" in cap.size)
|
|
238
|
-
if do_load:
|
|
239
|
-
print(f" [small-load-test] attempting for {cap.id}")
|
|
240
|
-
load = try_test_load_small(cap.hf_repo_id)
|
|
241
|
-
time.sleep(0.6)
|
|
242
|
-
|
|
243
|
-
# Merge into verification view
|
|
244
|
-
merged = {**light}
|
|
245
|
-
if deep:
|
|
246
|
-
merged["deep"] = deep
|
|
247
|
-
if deep.get("has_config"):
|
|
248
|
-
merged["has_config_hint"] = True
|
|
249
|
-
if deep.get("has_tokenizer"):
|
|
250
|
-
merged["has_tokenizer_hint"] = True
|
|
251
|
-
if load:
|
|
252
|
-
merged["load_test"] = load
|
|
253
|
-
|
|
254
|
-
if not merged["hf_exists"]:
|
|
255
|
-
if cap.recommended_default:
|
|
256
|
-
missing_critical += 1
|
|
257
|
-
merged["notes"] = (merged.get("notes") or "") + " CRITICAL: missing from HF!"
|
|
258
|
-
|
|
259
|
-
# Pretty line
|
|
260
|
-
status = "✓" if merged["hf_exists"] else "✗"
|
|
261
|
-
v = "V" if merged.get("has_config_hint") and merged.get("has_tokenizer_hint") else "?"
|
|
262
|
-
large = " LARGE" if is_large else ""
|
|
263
|
-
print(f"{status} {cap.id:<52} {cap.size:>8} {cap.family:<14} {v} {large}")
|
|
264
|
-
|
|
265
|
-
results.append(merged)
|
|
282
|
+
for cap in sorted(caps, key=lambda c: (c.lifecycle != RECOMMENDED, c.display_priority, c.id)):
|
|
283
|
+
result = verify_one(cap)
|
|
284
|
+
_print_row(result)
|
|
285
|
+
results.append(result)
|
|
286
|
+
|
|
287
|
+
recommended = [r for r in results if r["lifecycle"] == RECOMMENDED]
|
|
288
|
+
legacy = [r for r in results if r["lifecycle"] != RECOMMENDED]
|
|
289
|
+
failing = [r for r in recommended if r["verdict"] != VERDICT_LOADABLE]
|
|
266
290
|
|
|
267
291
|
summary = {
|
|
268
292
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
269
293
|
"total": len(results),
|
|
270
|
-
"
|
|
271
|
-
"
|
|
272
|
-
"
|
|
273
|
-
"
|
|
274
|
-
"
|
|
275
|
-
"
|
|
294
|
+
"recommended_total": len(recommended),
|
|
295
|
+
"legacy_total": len(legacy),
|
|
296
|
+
"hf_present": sum(1 for r in results if r["hf_exists"]),
|
|
297
|
+
"loadable_static": sum(1 for r in results if r["verdict"] == VERDICT_LOADABLE),
|
|
298
|
+
"needs_review": sum(1 for r in results if r["verdict"] == VERDICT_NEEDS_REVIEW),
|
|
299
|
+
"unavailable": sum(1 for r in results if r["verdict"] == VERDICT_UNAVAILABLE),
|
|
300
|
+
"recommended_failing": [r["id"] for r in failing],
|
|
301
|
+
"weights_downloaded": 0,
|
|
302
|
+
"models_loaded": 0,
|
|
276
303
|
}
|
|
277
304
|
|
|
278
305
|
report = {
|
|
279
306
|
"summary": summary,
|
|
307
|
+
"verdict_criteria": {
|
|
308
|
+
"loadable_static": [
|
|
309
|
+
"repo reachable anonymously and not gated",
|
|
310
|
+
"registry id matches the Hub's canonical id exactly (including case)",
|
|
311
|
+
"library_name == 'mlx' or an 'mlx' tag is present",
|
|
312
|
+
"config model_type is in SUPPORTED_MLX_ARCHITECTURES and matches the registry",
|
|
313
|
+
"file tree has config.json, a tokenizer file and >=1 .safetensors shard",
|
|
314
|
+
f"at least {MIN_COMMUNITY_DOWNLOADS} all-time downloads",
|
|
315
|
+
"measured sibling byte sum is within 0.2GB of the registry's download_size_gb",
|
|
316
|
+
],
|
|
317
|
+
"supported_mlx_architectures": SUPPORTED_MLX_ARCHITECTURES,
|
|
318
|
+
"min_community_downloads": MIN_COMMUNITY_DOWNLOADS,
|
|
319
|
+
},
|
|
320
|
+
"limitations": LIMITATIONS,
|
|
280
321
|
"results": results,
|
|
281
|
-
"recommendation": "All primary recommended models are present on HF with config+tokenizer hints. "
|
|
282
|
-
"Large models (>12GB) have explicit LOCAL_LOAD_LIMITED notes. "
|
|
283
|
-
"Use --deep or --test-load only when you have huggingface_hub/transformers and want to exercise small-model paths. "
|
|
284
|
-
"Never use this script to pre-download production weights; respect user consent.",
|
|
285
322
|
}
|
|
286
323
|
|
|
287
324
|
out_path = Path(args.out).resolve()
|
|
288
325
|
out_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
289
|
-
print("-" * 88)
|
|
290
|
-
print(json.dumps(summary, indent=2))
|
|
291
|
-
print(f"\nFull report written: {out_path}")
|
|
292
326
|
|
|
293
|
-
|
|
294
|
-
print(
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
327
|
+
print("-" * 96)
|
|
328
|
+
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
|
329
|
+
print("\nLimitations of this verdict:")
|
|
330
|
+
for line in LIMITATIONS:
|
|
331
|
+
print(f" · {line}")
|
|
332
|
+
print(f"\nFull report written: {out_path}")
|
|
298
333
|
|
|
299
|
-
if
|
|
300
|
-
print(f"\n**FAIL**: {
|
|
334
|
+
if failing:
|
|
335
|
+
print(f"\n**FAIL**: {len(failing)} recommended entries are not statically loadable.")
|
|
301
336
|
return 1
|
|
302
337
|
return 0
|
|
303
338
|
|
package/src-tauri/Cargo.lock
CHANGED
package/src-tauri/Cargo.toml
CHANGED