nexo-brain 7.9.13 → 7.9.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/cognitive/_core.py +6 -4
- package/src/db/_protocol.py +10 -1
- package/src/db/_schema.py +9 -0
- package/src/doctor/providers/runtime.py +22 -4
- package/src/local_model_manifest.json +113 -0
- package/src/local_models.py +247 -0
- package/src/migrate_embeddings.py +6 -6
- package/src/model_warmup.py +20 -23
- package/src/paths.py +9 -0
- package/src/plugins/cortex.py +267 -34
- package/src/plugins/protocol.py +125 -47
- package/src/resonance_map.py +2 -0
- package/src/scripts/nexo-daily-self-audit.py +44 -0
- package/src/scripts/nexo-learning-housekeep.py +2 -2
- package/templates/core-prompts/cortex-decision-critic.md +24 -0
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
[Watch the overview video](https://nexo-brain.com/watch/) · [Watch on YouTube](https://www.youtube.com/watch?v=i2lkGhKyVqI) · [Open the infographic](https://nexo-brain.com/assets/nexo-brain-infographic-v5.png)
|
|
20
20
|
|
|
21
|
-
Version `7.9.
|
|
21
|
+
Version `7.9.14` is the current packaged-runtime line. Patch release over `7.9.13`: `task_close(done)` now hard-blocks missing verify/change-log/cortex evidence instead of silently degrading to debt-only closes, self-audit auto-drains stale `protocol_debt` every day, and Codex session parity now flags partial bootstrap/startup/heartbeat drift instead of passing as healthy when only one recent session behaved correctly. Coordinated Desktop release remains v0.28.14.
|
|
22
22
|
|
|
23
23
|
Previously in `7.9.5`: patch release that fixes canonical diary confirmation for Desktop: Brain resolves the Desktop/Claude session UUID through NEXO SID aliases before checking `session_diary`, so archive/delete/app-exit can confirm diaries written by `nexo_session_diary_write` under the active `nexo-...` SID. Verification: `pytest tests/test_lifecycle_events.py` (28 passing) plus coordinated Desktop v0.28.6 shutdown/archive/delete/app-exit checks.
|
|
24
24
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.9.
|
|
3
|
+
"version": "7.9.15",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/cognitive/_core.py
CHANGED
|
@@ -558,8 +558,9 @@ def _get_model():
|
|
|
558
558
|
"""Lazy-load fastembed TextEmbedding model."""
|
|
559
559
|
global _model
|
|
560
560
|
if _model is None:
|
|
561
|
-
from
|
|
562
|
-
|
|
561
|
+
from local_models import build_fastembed_embedding
|
|
562
|
+
|
|
563
|
+
_model = build_fastembed_embedding("bge-base-embeddings")
|
|
563
564
|
return _model
|
|
564
565
|
|
|
565
566
|
|
|
@@ -568,8 +569,9 @@ def _get_reranker():
|
|
|
568
569
|
global _reranker
|
|
569
570
|
if _reranker is None:
|
|
570
571
|
try:
|
|
571
|
-
from
|
|
572
|
-
|
|
572
|
+
from local_models import build_fastembed_reranker
|
|
573
|
+
|
|
574
|
+
_reranker = build_fastembed_reranker("cross-encoder-reranker")
|
|
573
575
|
except Exception:
|
|
574
576
|
_reranker = False # Mark as unavailable
|
|
575
577
|
return _reranker if _reranker is not False else None
|
package/src/db/_protocol.py
CHANGED
|
@@ -195,6 +195,10 @@ def create_cortex_evaluation(
|
|
|
195
195
|
goal_profile_id: str = "",
|
|
196
196
|
goal_profile_labels=None,
|
|
197
197
|
goal_profile_weights=None,
|
|
198
|
+
heuristic_choice: str = "",
|
|
199
|
+
heuristic_reasoning: str = "",
|
|
200
|
+
critique_payload=None,
|
|
201
|
+
decision_mode: str = "heuristic",
|
|
198
202
|
selected_choice: str = "",
|
|
199
203
|
selection_reason: str = "",
|
|
200
204
|
selection_source: str = "recommended",
|
|
@@ -206,8 +210,9 @@ def create_cortex_evaluation(
|
|
|
206
210
|
session_id, task_id, goal, task_type, area, impact_level, context_hint,
|
|
207
211
|
alternatives, scores, recommended_choice, recommended_reasoning, linked_outcome_id,
|
|
208
212
|
goal_profile_id, goal_profile_labels, goal_profile_weights,
|
|
213
|
+
heuristic_choice, heuristic_reasoning, critique_payload, decision_mode,
|
|
209
214
|
selected_choice, selection_reason, selection_source
|
|
210
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
215
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
211
216
|
(
|
|
212
217
|
session_id.strip(),
|
|
213
218
|
task_id.strip(),
|
|
@@ -224,6 +229,10 @@ def create_cortex_evaluation(
|
|
|
224
229
|
goal_profile_id.strip(),
|
|
225
230
|
_as_json(goal_profile_labels or []),
|
|
226
231
|
_as_json(goal_profile_weights or {}),
|
|
232
|
+
(heuristic_choice or recommended_choice).strip(),
|
|
233
|
+
(heuristic_reasoning or recommended_reasoning).strip(),
|
|
234
|
+
_as_json(critique_payload or {}),
|
|
235
|
+
(decision_mode or "heuristic").strip(),
|
|
227
236
|
(selected_choice or recommended_choice).strip(),
|
|
228
237
|
(selection_reason or recommended_reasoning).strip(),
|
|
229
238
|
(selection_source or "recommended").strip(),
|
package/src/db/_schema.py
CHANGED
|
@@ -909,6 +909,14 @@ def _m38_evolution_log_proposal_payload(conn):
|
|
|
909
909
|
_migrate_add_column(conn, "evolution_log", "proposal_payload", "TEXT DEFAULT NULL")
|
|
910
910
|
|
|
911
911
|
|
|
912
|
+
def _m55_cortex_critique_trace(conn):
|
|
913
|
+
"""Persist heuristic-vs-LLM critique traces for Cortex decisions."""
|
|
914
|
+
_migrate_add_column(conn, "cortex_evaluations", "heuristic_choice", "TEXT DEFAULT ''")
|
|
915
|
+
_migrate_add_column(conn, "cortex_evaluations", "heuristic_reasoning", "TEXT DEFAULT ''")
|
|
916
|
+
_migrate_add_column(conn, "cortex_evaluations", "critique_payload", "TEXT DEFAULT '{}'")
|
|
917
|
+
_migrate_add_column(conn, "cortex_evaluations", "decision_mode", "TEXT DEFAULT 'heuristic'")
|
|
918
|
+
|
|
919
|
+
|
|
912
920
|
def _m39_hook_runs(conn):
|
|
913
921
|
"""Persist hook lifecycle observability — closes Fase 3 item 7.
|
|
914
922
|
|
|
@@ -1479,6 +1487,7 @@ MIGRATIONS = [
|
|
|
1479
1487
|
(52, "lifecycle_canonical_plan", _m52_lifecycle_canonical_plan),
|
|
1480
1488
|
(53, "session_conversation_identity", _m53_session_conversation_identity),
|
|
1481
1489
|
(54, "continuity_snapshots", _m54_continuity_snapshots),
|
|
1490
|
+
(55, "cortex_critique_trace", _m55_cortex_critique_trace),
|
|
1482
1491
|
]
|
|
1483
1492
|
|
|
1484
1493
|
|
|
@@ -2112,14 +2112,32 @@ def check_codex_session_parity() -> DoctorCheck:
|
|
|
2112
2112
|
status = "healthy"
|
|
2113
2113
|
severity = "info"
|
|
2114
2114
|
repair_plan: list[str] = []
|
|
2115
|
-
|
|
2115
|
+
missing_bootstrap = max(0, audit["files"] - audit["bootstrap_sessions"])
|
|
2116
|
+
missing_startup = max(0, audit["files"] - audit["startup_sessions"])
|
|
2117
|
+
missing_heartbeat = max(0, audit["files"] - audit["heartbeat_sessions"])
|
|
2118
|
+
if missing_bootstrap:
|
|
2116
2119
|
status = "degraded"
|
|
2117
2120
|
severity = "warn"
|
|
2118
|
-
repair_plan.append(
|
|
2119
|
-
|
|
2121
|
+
repair_plan.append(
|
|
2122
|
+
"Run `nexo update` or `nexo clients sync` so every Codex session inherits the managed bootstrap, not just a subset"
|
|
2123
|
+
)
|
|
2124
|
+
if missing_startup:
|
|
2125
|
+
status = "degraded"
|
|
2126
|
+
severity = "warn"
|
|
2127
|
+
repair_plan.append(
|
|
2128
|
+
"Use `nexo chat` or keep the global Codex bootstrap intact so every Codex session actually calls `nexo_startup`"
|
|
2129
|
+
)
|
|
2130
|
+
if missing_heartbeat:
|
|
2120
2131
|
status = "degraded"
|
|
2121
2132
|
severity = "warn"
|
|
2122
|
-
repair_plan.append("
|
|
2133
|
+
repair_plan.append("Keep `nexo_heartbeat` on every user turn so restored/plain Codex sessions do not drift off-protocol")
|
|
2134
|
+
if missing_bootstrap or missing_startup or missing_heartbeat:
|
|
2135
|
+
evidence.append(
|
|
2136
|
+
"session drift: "
|
|
2137
|
+
f"{missing_bootstrap} missing bootstrap, "
|
|
2138
|
+
f"{missing_startup} missing startup, "
|
|
2139
|
+
f"{missing_heartbeat} missing heartbeat"
|
|
2140
|
+
)
|
|
2123
2141
|
|
|
2124
2142
|
return DoctorCheck(
|
|
2125
2143
|
id="runtime.codex_sessions",
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"models": [
|
|
4
|
+
{
|
|
5
|
+
"name": "bge-base-embeddings",
|
|
6
|
+
"kind": "fastembed_embedding",
|
|
7
|
+
"model_id": "BAAI/bge-base-en-v1.5",
|
|
8
|
+
"source_repo": "qdrant/bge-base-en-v1.5-onnx-q",
|
|
9
|
+
"revision": "738cad1c108e2f23649db9e44b2eab988626493b",
|
|
10
|
+
"model_file": "model_optimized.onnx",
|
|
11
|
+
"source": "src/cognitive/_core.py",
|
|
12
|
+
"required_files": [
|
|
13
|
+
{
|
|
14
|
+
"path": "config.json",
|
|
15
|
+
"size": 740,
|
|
16
|
+
"sha256": "86f84a5285de7f1ee673f712387219ef1e261ec27dcd870e793a80f9da1aaa3b"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"path": "model_optimized.onnx",
|
|
20
|
+
"size": 217824172,
|
|
21
|
+
"sha256": "4e556722bc4f65716c544c8a931f1e90fb3f866e5741fd93a96f051d673339c7"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"path": "special_tokens_map.json",
|
|
25
|
+
"size": 695,
|
|
26
|
+
"sha256": "5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"path": "tokenizer.json",
|
|
30
|
+
"size": 711396,
|
|
31
|
+
"sha256": "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"path": "tokenizer_config.json",
|
|
35
|
+
"size": 1242,
|
|
36
|
+
"sha256": "0b29c7bfc889e53b36d9dd3e686dd4300f6525110eaa98c76a5dafceb2029f53"
|
|
37
|
+
}
|
|
38
|
+
]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"name": "bge-small-embeddings",
|
|
42
|
+
"kind": "fastembed_embedding",
|
|
43
|
+
"model_id": "BAAI/bge-small-en-v1.5",
|
|
44
|
+
"source_repo": "qdrant/bge-small-en-v1.5-onnx-q",
|
|
45
|
+
"revision": "52398278842ec682c6f32300af41344b1c0b0bb2",
|
|
46
|
+
"model_file": "model_optimized.onnx",
|
|
47
|
+
"source": "src/migrate_embeddings.py",
|
|
48
|
+
"required_files": [
|
|
49
|
+
{
|
|
50
|
+
"path": "config.json",
|
|
51
|
+
"size": 706,
|
|
52
|
+
"sha256": "13582bcf2effc85b7bf3d3f5532e686bc1c9ce86bb009d10f0ec33cbe92299dd"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"path": "model_optimized.onnx",
|
|
56
|
+
"size": 66465124,
|
|
57
|
+
"sha256": "51f1bd0addd6e859e42c2c8021a5e5461385bb676a649f4b269aa445449f2431"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"path": "special_tokens_map.json",
|
|
61
|
+
"size": 695,
|
|
62
|
+
"sha256": "5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"path": "tokenizer.json",
|
|
66
|
+
"size": 711396,
|
|
67
|
+
"sha256": "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"path": "tokenizer_config.json",
|
|
71
|
+
"size": 1242,
|
|
72
|
+
"sha256": "0b29c7bfc889e53b36d9dd3e686dd4300f6525110eaa98c76a5dafceb2029f53"
|
|
73
|
+
}
|
|
74
|
+
]
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"name": "cross-encoder-reranker",
|
|
78
|
+
"kind": "fastembed_reranker",
|
|
79
|
+
"model_id": "Xenova/ms-marco-MiniLM-L-6-v2",
|
|
80
|
+
"source_repo": "Xenova/ms-marco-MiniLM-L-6-v2",
|
|
81
|
+
"revision": "a09144355adeed5f58c8ed011d209bf8ee5a1fec",
|
|
82
|
+
"model_file": "onnx/model.onnx",
|
|
83
|
+
"source": "src/cognitive/_core.py",
|
|
84
|
+
"required_files": [
|
|
85
|
+
{
|
|
86
|
+
"path": "config.json",
|
|
87
|
+
"size": 824,
|
|
88
|
+
"sha256": "d827779a72d27ae68cf878a6fc2e954542663fe21ca515d9f4783fc96be2d37e"
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"path": "onnx/model.onnx",
|
|
92
|
+
"size": 90992115,
|
|
93
|
+
"sha256": "c623d0bcb99f4622beb413eaef00cfbe5db20df9f1dd982da4b4f26022881870"
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"path": "special_tokens_map.json",
|
|
97
|
+
"size": 125,
|
|
98
|
+
"sha256": "b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"path": "tokenizer.json",
|
|
102
|
+
"size": 711396,
|
|
103
|
+
"sha256": "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"path": "tokenizer_config.json",
|
|
107
|
+
"size": 1242,
|
|
108
|
+
"sha256": "0b29c7bfc889e53b36d9dd3e686dd4300f6525110eaa98c76a5dafceb2029f53"
|
|
109
|
+
}
|
|
110
|
+
]
|
|
111
|
+
}
|
|
112
|
+
]
|
|
113
|
+
}
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Pinned local model management for Brain embeddings + reranker.
|
|
2
|
+
|
|
3
|
+
FastEmbed's built-in registry resolves supported models by friendly name, but
|
|
4
|
+
its downloader tracks the current upstream repo head unless the caller builds a
|
|
5
|
+
stronger contract around it. This module provides that stronger contract:
|
|
6
|
+
|
|
7
|
+
- source repo pinned by immutable revision SHA
|
|
8
|
+
- required files + sha256 checksums stored in-repo
|
|
9
|
+
- deterministic materialization under ``~/.nexo/runtime/models``
|
|
10
|
+
- FastEmbed instantiated via ``specific_model_path`` so runtime loads the exact
|
|
11
|
+
downloaded artifacts instead of following a floating registry download
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import tempfile
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from functools import lru_cache
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
import paths
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
MANIFEST_PATH = Path(__file__).resolve().with_name("local_model_manifest.json")
|
|
31
|
+
MODEL_LOCK_FILENAME = ".nexo-model-lock.json"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class LocalModelFile:
|
|
36
|
+
path: str
|
|
37
|
+
size: int
|
|
38
|
+
sha256: str
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class LocalModelSpec:
|
|
43
|
+
name: str
|
|
44
|
+
kind: str
|
|
45
|
+
model_id: str
|
|
46
|
+
source_repo: str
|
|
47
|
+
revision: str
|
|
48
|
+
model_file: str
|
|
49
|
+
source: str
|
|
50
|
+
required_files: tuple[LocalModelFile, ...]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _slugify(value: str) -> str:
|
|
54
|
+
return re.sub(r"[^a-z0-9]+", "-", value.strip().lower()).strip("-")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _hash_file(path: Path) -> str:
|
|
58
|
+
digest = hashlib.sha256()
|
|
59
|
+
with path.open("rb") as handle:
|
|
60
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
61
|
+
digest.update(chunk)
|
|
62
|
+
return digest.hexdigest()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _lock_payload(spec: LocalModelSpec) -> dict[str, Any]:
|
|
66
|
+
return {
|
|
67
|
+
"name": spec.name,
|
|
68
|
+
"kind": spec.kind,
|
|
69
|
+
"model_id": spec.model_id,
|
|
70
|
+
"source_repo": spec.source_repo,
|
|
71
|
+
"revision": spec.revision,
|
|
72
|
+
"model_file": spec.model_file,
|
|
73
|
+
"required_files": [
|
|
74
|
+
{"path": item.path, "size": item.size, "sha256": item.sha256}
|
|
75
|
+
for item in spec.required_files
|
|
76
|
+
],
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@lru_cache(maxsize=1)
|
|
81
|
+
def _load_manifest() -> dict[str, LocalModelSpec]:
|
|
82
|
+
payload = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
|
|
83
|
+
specs: dict[str, LocalModelSpec] = {}
|
|
84
|
+
for raw in payload.get("models", []) or []:
|
|
85
|
+
files = tuple(LocalModelFile(**item) for item in raw.get("required_files", []) or [])
|
|
86
|
+
spec = LocalModelSpec(
|
|
87
|
+
name=str(raw["name"]),
|
|
88
|
+
kind=str(raw["kind"]),
|
|
89
|
+
model_id=str(raw["model_id"]),
|
|
90
|
+
source_repo=str(raw["source_repo"]),
|
|
91
|
+
revision=str(raw["revision"]),
|
|
92
|
+
model_file=str(raw["model_file"]),
|
|
93
|
+
source=str(raw["source"]),
|
|
94
|
+
required_files=files,
|
|
95
|
+
)
|
|
96
|
+
specs[spec.name] = spec
|
|
97
|
+
return specs
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def get_local_model_spec(name: str) -> LocalModelSpec:
|
|
101
|
+
try:
|
|
102
|
+
return _load_manifest()[name]
|
|
103
|
+
except KeyError as exc: # pragma: no cover - defensive
|
|
104
|
+
raise KeyError(f"unknown local model spec: {name}") from exc
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def list_local_model_specs(kind: str | None = None) -> list[LocalModelSpec]:
|
|
108
|
+
specs = list(_load_manifest().values())
|
|
109
|
+
if kind:
|
|
110
|
+
specs = [spec for spec in specs if spec.kind == kind]
|
|
111
|
+
return specs
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def models_dir() -> Path:
|
|
115
|
+
root = paths.models_dir()
|
|
116
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
117
|
+
return root
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def managed_model_dir(spec: LocalModelSpec) -> Path:
|
|
121
|
+
return models_dir() / _slugify(spec.name) / spec.revision
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def verify_local_model_dir(spec: LocalModelSpec, root: Path | None = None) -> dict[str, Any]:
|
|
125
|
+
target = root or managed_model_dir(spec)
|
|
126
|
+
problems: list[str] = []
|
|
127
|
+
if not target.exists():
|
|
128
|
+
problems.append("missing directory")
|
|
129
|
+
for file_spec in spec.required_files:
|
|
130
|
+
file_path = target / file_spec.path
|
|
131
|
+
if not file_path.exists():
|
|
132
|
+
problems.append(f"missing file:{file_spec.path}")
|
|
133
|
+
continue
|
|
134
|
+
size = file_path.stat().st_size
|
|
135
|
+
if size != file_spec.size:
|
|
136
|
+
problems.append(f"size mismatch:{file_spec.path}:{size}!={file_spec.size}")
|
|
137
|
+
continue
|
|
138
|
+
actual_hash = _hash_file(file_path)
|
|
139
|
+
if actual_hash != file_spec.sha256:
|
|
140
|
+
problems.append(f"sha256 mismatch:{file_spec.path}:{actual_hash}!={file_spec.sha256}")
|
|
141
|
+
lock_path = target / MODEL_LOCK_FILENAME
|
|
142
|
+
if lock_path.exists():
|
|
143
|
+
try:
|
|
144
|
+
payload = json.loads(lock_path.read_text(encoding="utf-8"))
|
|
145
|
+
if payload.get("revision") != spec.revision or payload.get("source_repo") != spec.source_repo:
|
|
146
|
+
problems.append("lock metadata mismatch")
|
|
147
|
+
except Exception:
|
|
148
|
+
problems.append("invalid lock metadata")
|
|
149
|
+
return {"ok": not problems, "path": str(target), "problems": problems}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _copy_required_files(snapshot_dir: Path, target_dir: Path, spec: LocalModelSpec) -> None:
|
|
153
|
+
for file_spec in spec.required_files:
|
|
154
|
+
source_path = snapshot_dir / file_spec.path
|
|
155
|
+
if not source_path.exists():
|
|
156
|
+
raise FileNotFoundError(f"snapshot missing required file: {file_spec.path}")
|
|
157
|
+
destination = target_dir / file_spec.path
|
|
158
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
shutil.copy2(source_path, destination)
|
|
160
|
+
(target_dir / MODEL_LOCK_FILENAME).write_text(
|
|
161
|
+
json.dumps(_lock_payload(spec), indent=2, ensure_ascii=False) + "\n",
|
|
162
|
+
encoding="utf-8",
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def ensure_local_model(
|
|
167
|
+
name: str,
|
|
168
|
+
*,
|
|
169
|
+
local_files_only: bool = False,
|
|
170
|
+
force_redownload: bool = False,
|
|
171
|
+
) -> Path:
|
|
172
|
+
spec = get_local_model_spec(name)
|
|
173
|
+
target_dir = managed_model_dir(spec)
|
|
174
|
+
verification = verify_local_model_dir(spec, target_dir)
|
|
175
|
+
if verification["ok"] and not force_redownload:
|
|
176
|
+
return target_dir
|
|
177
|
+
if target_dir.exists():
|
|
178
|
+
shutil.rmtree(target_dir, ignore_errors=True)
|
|
179
|
+
|
|
180
|
+
from huggingface_hub import snapshot_download
|
|
181
|
+
|
|
182
|
+
cache_dir = models_dir() / "_hf-cache"
|
|
183
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
184
|
+
snapshot_dir = Path(
|
|
185
|
+
snapshot_download(
|
|
186
|
+
repo_id=spec.source_repo,
|
|
187
|
+
revision=spec.revision,
|
|
188
|
+
allow_patterns=[item.path for item in spec.required_files],
|
|
189
|
+
cache_dir=str(cache_dir),
|
|
190
|
+
local_files_only=local_files_only,
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
target_parent = target_dir.parent
|
|
195
|
+
target_parent.mkdir(parents=True, exist_ok=True)
|
|
196
|
+
tmp_dir = Path(
|
|
197
|
+
tempfile.mkdtemp(
|
|
198
|
+
prefix=f".{_slugify(spec.name)}-{spec.revision[:12]}.",
|
|
199
|
+
dir=str(target_parent),
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
try:
|
|
203
|
+
_copy_required_files(snapshot_dir, tmp_dir, spec)
|
|
204
|
+
verification = verify_local_model_dir(spec, tmp_dir)
|
|
205
|
+
if not verification["ok"]:
|
|
206
|
+
raise ValueError("; ".join(verification["problems"]))
|
|
207
|
+
os.replace(tmp_dir, target_dir)
|
|
208
|
+
except Exception:
|
|
209
|
+
shutil.rmtree(tmp_dir, ignore_errors=True)
|
|
210
|
+
raise
|
|
211
|
+
return target_dir
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def build_fastembed_embedding(name: str):
|
|
215
|
+
spec = get_local_model_spec(name)
|
|
216
|
+
if spec.kind != "fastembed_embedding":
|
|
217
|
+
raise ValueError(f"{name} is not a fastembed embedding model")
|
|
218
|
+
from fastembed import TextEmbedding
|
|
219
|
+
|
|
220
|
+
target_dir = ensure_local_model(name)
|
|
221
|
+
return TextEmbedding(spec.model_id, specific_model_path=str(target_dir))
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def build_fastembed_reranker(name: str):
|
|
225
|
+
spec = get_local_model_spec(name)
|
|
226
|
+
if spec.kind != "fastembed_reranker":
|
|
227
|
+
raise ValueError(f"{name} is not a fastembed reranker")
|
|
228
|
+
from fastembed.rerank.cross_encoder import TextCrossEncoder
|
|
229
|
+
|
|
230
|
+
target_dir = ensure_local_model(name)
|
|
231
|
+
return TextCrossEncoder(spec.model_id, specific_model_path=str(target_dir))
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
__all__ = [
|
|
235
|
+
"LocalModelFile",
|
|
236
|
+
"LocalModelSpec",
|
|
237
|
+
"MODEL_LOCK_FILENAME",
|
|
238
|
+
"MANIFEST_PATH",
|
|
239
|
+
"build_fastembed_embedding",
|
|
240
|
+
"build_fastembed_reranker",
|
|
241
|
+
"ensure_local_model",
|
|
242
|
+
"get_local_model_spec",
|
|
243
|
+
"list_local_model_specs",
|
|
244
|
+
"managed_model_dir",
|
|
245
|
+
"models_dir",
|
|
246
|
+
"verify_local_model_dir",
|
|
247
|
+
]
|
|
@@ -16,6 +16,7 @@ import time
|
|
|
16
16
|
import numpy as np
|
|
17
17
|
|
|
18
18
|
import paths
|
|
19
|
+
from local_models import build_fastembed_embedding, get_local_model_spec
|
|
19
20
|
|
|
20
21
|
NEXO_HOME = os.environ.get("NEXO_HOME", os.path.expanduser("~/.nexo"))
|
|
21
22
|
_cognitive_dir = paths.cognitive_dir()
|
|
@@ -24,8 +25,8 @@ DB_PATH = str(_cognitive_dir / "cognitive.db")
|
|
|
24
25
|
BACKUP_PATH = DB_PATH + ".bak-384dims-pre-upgrade"
|
|
25
26
|
|
|
26
27
|
MODELS = {
|
|
27
|
-
"small": ("
|
|
28
|
-
"base": ("
|
|
28
|
+
"small": ("bge-small-embeddings", 384),
|
|
29
|
+
"base": ("bge-base-embeddings", 768),
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
|
|
@@ -47,8 +48,6 @@ def verify():
|
|
|
47
48
|
|
|
48
49
|
def upgrade():
|
|
49
50
|
"""Re-embed all memories from bge-small (384) to bge-base (768)."""
|
|
50
|
-
from fastembed import TextEmbedding
|
|
51
|
-
|
|
52
51
|
# Verify current state
|
|
53
52
|
print("Current state:")
|
|
54
53
|
verify()
|
|
@@ -62,8 +61,9 @@ def upgrade():
|
|
|
62
61
|
|
|
63
62
|
# Load new model
|
|
64
63
|
model_name, expected_dim = MODELS["base"]
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
spec = get_local_model_spec(model_name)
|
|
65
|
+
print(f"\nLoading {spec.model_id}@{spec.revision}...")
|
|
66
|
+
model = build_fastembed_embedding(model_name)
|
|
67
67
|
|
|
68
68
|
conn = sqlite3.connect(DB_PATH)
|
|
69
69
|
try:
|
package/src/model_warmup.py
CHANGED
|
@@ -28,40 +28,37 @@ class WarmupTarget:
|
|
|
28
28
|
kind: str
|
|
29
29
|
model_id: str
|
|
30
30
|
source: str
|
|
31
|
+
source_repo: str | None = None
|
|
31
32
|
revision: str | None = None
|
|
32
33
|
required: bool = True
|
|
33
34
|
|
|
34
35
|
|
|
35
36
|
def warmup_targets() -> list[WarmupTarget]:
|
|
36
37
|
from classifier_local import MODEL_ID, MODEL_REVISION
|
|
38
|
+
from local_models import list_local_model_specs
|
|
37
39
|
|
|
38
|
-
|
|
40
|
+
targets = [
|
|
39
41
|
WarmupTarget(
|
|
40
42
|
name="local-zero-shot-classifier",
|
|
41
43
|
kind="transformers_sequence_classifier",
|
|
42
44
|
model_id=MODEL_ID,
|
|
43
45
|
revision=MODEL_REVISION,
|
|
44
46
|
source="src/classifier_local.py",
|
|
45
|
-
|
|
46
|
-
WarmupTarget(
|
|
47
|
-
name="bge-base-embeddings",
|
|
48
|
-
kind="fastembed_embedding",
|
|
49
|
-
model_id="BAAI/bge-base-en-v1.5",
|
|
50
|
-
source="src/cognitive/_core.py",
|
|
51
|
-
),
|
|
52
|
-
WarmupTarget(
|
|
53
|
-
name="bge-small-embeddings",
|
|
54
|
-
kind="fastembed_embedding",
|
|
55
|
-
model_id="BAAI/bge-small-en-v1.5",
|
|
56
|
-
source="src/migrate_embeddings.py",
|
|
57
|
-
),
|
|
58
|
-
WarmupTarget(
|
|
59
|
-
name="cross-encoder-reranker",
|
|
60
|
-
kind="fastembed_reranker",
|
|
61
|
-
model_id="Xenova/ms-marco-MiniLM-L-6-v2",
|
|
62
|
-
source="src/cognitive/_core.py",
|
|
47
|
+
source_repo=MODEL_ID,
|
|
63
48
|
),
|
|
64
49
|
]
|
|
50
|
+
for spec in list_local_model_specs():
|
|
51
|
+
targets.append(
|
|
52
|
+
WarmupTarget(
|
|
53
|
+
name=spec.name,
|
|
54
|
+
kind=spec.kind,
|
|
55
|
+
model_id=spec.model_id,
|
|
56
|
+
revision=spec.revision,
|
|
57
|
+
source=spec.source,
|
|
58
|
+
source_repo=spec.source_repo,
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
return targets
|
|
65
62
|
|
|
66
63
|
|
|
67
64
|
def _state_path() -> Path:
|
|
@@ -86,16 +83,16 @@ def _warm_transformers(target: WarmupTarget) -> None:
|
|
|
86
83
|
|
|
87
84
|
|
|
88
85
|
def _warm_fastembed_embedding(target: WarmupTarget) -> None:
|
|
89
|
-
from
|
|
86
|
+
from local_models import build_fastembed_embedding
|
|
90
87
|
|
|
91
|
-
model =
|
|
88
|
+
model = build_fastembed_embedding(target.name)
|
|
92
89
|
list(model.embed(["NEXO model warmup"]))
|
|
93
90
|
|
|
94
91
|
|
|
95
92
|
def _warm_fastembed_reranker(target: WarmupTarget) -> None:
|
|
96
|
-
from
|
|
93
|
+
from local_models import build_fastembed_reranker
|
|
97
94
|
|
|
98
|
-
|
|
95
|
+
build_fastembed_reranker(target.name)
|
|
99
96
|
|
|
100
97
|
|
|
101
98
|
def warm_target(target: WarmupTarget) -> None:
|
package/src/paths.py
CHANGED
|
@@ -336,6 +336,14 @@ def cognitive_dir() -> Path:
|
|
|
336
336
|
return new
|
|
337
337
|
|
|
338
338
|
|
|
339
|
+
def models_dir() -> Path:
|
|
340
|
+
new = runtime_dir() / "models"
|
|
341
|
+
legacy = home() / "models"
|
|
342
|
+
if not new.exists() and legacy.exists():
|
|
343
|
+
return legacy
|
|
344
|
+
return new
|
|
345
|
+
|
|
346
|
+
|
|
339
347
|
def coordination_dir() -> Path:
|
|
340
348
|
new = runtime_dir() / "coordination"
|
|
341
349
|
legacy = home() / "coordination"
|
|
@@ -488,6 +496,7 @@ __all__ = [
|
|
|
488
496
|
"backups_dir",
|
|
489
497
|
"memory_dir",
|
|
490
498
|
"cognitive_dir",
|
|
499
|
+
"models_dir",
|
|
491
500
|
"coordination_dir",
|
|
492
501
|
"exports_dir",
|
|
493
502
|
"nexo_email_dir",
|