nexo-brain 7.14.0 → 7.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +6 -4
- package/bin/nexo-brain.js +27 -6
- package/package.json +1 -1
- package/src/agent_runner.py +86 -0
- package/src/claim_graph.py +19 -4
- package/src/cognitive/_core.py +124 -12
- package/src/cognitive/_search.py +156 -0
- package/src/db/_learnings.py +22 -10
- package/src/db/_semantic_similarity.py +4 -0
- package/src/doctor/providers/runtime.py +70 -0
- package/src/email_sent_events.py +14 -3
- package/src/enforcement_engine.py +52 -0
- package/src/hnsw_index.py +15 -3
- package/src/local_model_manifest.json +16 -13
- package/src/local_models.py +3 -0
- package/src/migrate_embeddings.py +17 -6
- package/src/plugins/cognitive_memory.py +1 -1
- package/src/scripts/nexo-daily-self-audit.py +129 -62
- package/src/scripts/nexo-email-monitor.py +163 -3
- package/src/scripts/nexo-send-reply.py +1 -0
- package/src/server.py +4 -2
- package/src/tools_learnings.py +37 -15
- package/templates/core-prompts/interactive-startup.md +1 -1
- package/templates/core-prompts/server-mcp-instructions.md +3 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.15.0",
|
|
4
4
|
"description": "Local cognitive runtime for Claude Code \u2014 persistent memory, overnight learning, doctor diagnostics, personal scripts, recovery-aware jobs, startup preflight, and optional dashboard/power helper.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "NEXO Brain",
|
package/README.md
CHANGED
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
|
|
19
19
|
[Watch the overview video](https://nexo-brain.com/watch/) · [Watch on YouTube](https://www.youtube.com/watch?v=i2lkGhKyVqI) · [Open the infographic](https://nexo-brain.com/assets/nexo-brain-infographic-v5.png)
|
|
20
20
|
|
|
21
|
-
Version `7.
|
|
21
|
+
Version `7.15.0` is the current packaged-runtime line. Minor release over v7.14.0 — Brain unifies sent-email continuity across send paths, moves cognitive recall to multilingual embeddings, forces tagged learnings into context, hardens email loop guards and headless runners, exposes learning creation dates, and adds AUTO-N burst postmortems.
|
|
22
|
+
|
|
23
|
+
Previously in `7.14.0`: minor release — Brain closes the install/reliability loop with update-path venv recovery, platform-gated wheels, WSL Desktop-managed flag preservation, startup memory authority warnings, legacy MEMORY write blocking, post-action real-world verification, and stale followup triage.
|
|
22
24
|
|
|
23
25
|
Previously in `7.13.9`: patch release — Brain moves aside an existing managed `.venv` when it was created with unsupported Python <3.10, then recreates it with the supported interpreter prepared by Desktop.
|
|
24
26
|
|
|
@@ -383,7 +385,7 @@ That keeps the core Ebbinghaus model, but makes decay more individual and less p
|
|
|
383
385
|
|
|
384
386
|
### Semantic Search (Finding by Meaning)
|
|
385
387
|
|
|
386
|
-
NEXO Brain doesn't search by keywords. It searches by **meaning** using vector embeddings (fastembed,
|
|
388
|
+
NEXO Brain doesn't search by keywords. It searches by **meaning** using multilingual vector embeddings (fastembed, 384 dimensions).
|
|
387
389
|
|
|
388
390
|
Example: If you search for "deploy problems", NEXO Brain will find a memory about "SSH connection timeout on production server" — even though they share zero words. This is how human associative memory works.
|
|
389
391
|
|
|
@@ -601,7 +603,7 @@ NEXO Brain was evaluated on [LoCoMo](https://github.com/snap-research/locomo) (A
|
|
|
601
603
|
- 93.3% adversarial rejection rate — reliably says "I don't know" when information isn't available
|
|
602
604
|
- 74.9% recall across 1,986 questions
|
|
603
605
|
- Open-domain F1: 0.637 | Multi-hop F1: 0.333 | Temporal F1: 0.326
|
|
604
|
-
- Runs on CPU with
|
|
606
|
+
- Runs on CPU with local multilingual embeddings — no GPU required
|
|
605
607
|
- First MCP memory server benchmarked on a peer-reviewed dataset
|
|
606
608
|
|
|
607
609
|
Full results in [`benchmarks/locomo/results/`](benchmarks/locomo/results/).
|
|
@@ -1447,7 +1449,7 @@ See [benchmarks/results/memory-recall-vs-static.md](benchmarks/results/memory-re
|
|
|
1447
1449
|
|
|
1448
1450
|
### v0.9.0 — Cognitive Memory (2026-03-15)
|
|
1449
1451
|
- Atkinson-Shiffrin memory model (STM → LTM promotion)
|
|
1450
|
-
- Semantic RAG with
|
|
1452
|
+
- Semantic RAG with pinned local multilingual fastembed models
|
|
1451
1453
|
- Trust scoring, sentiment detection, adaptive personality modes
|
|
1452
1454
|
- Ebbinghaus decay, sister detection, quarantine system
|
|
1453
1455
|
|
package/bin/nexo-brain.js
CHANGED
|
@@ -24,7 +24,26 @@ const readline = require("readline");
|
|
|
24
24
|
require = createRequire(path.join(__dirname, "nexo-brain.js"));
|
|
25
25
|
const { runViaWsl } = require("./windows-wsl-bridge");
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
function isCliEntrypoint() {
|
|
28
|
+
const invoked = process.argv && process.argv[1] ? String(process.argv[1]) : "";
|
|
29
|
+
if (!invoked) return false;
|
|
30
|
+
|
|
31
|
+
const normalize = (candidate) => {
|
|
32
|
+
try {
|
|
33
|
+
return fs.realpathSync.native(candidate);
|
|
34
|
+
} catch {
|
|
35
|
+
try {
|
|
36
|
+
return fs.realpathSync(candidate);
|
|
37
|
+
} catch {
|
|
38
|
+
return path.resolve(candidate);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
return normalize(invoked) === normalize(__filename);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (process.platform === "win32" && isCliEntrypoint()) {
|
|
28
47
|
const bridged = runViaWsl({
|
|
29
48
|
scriptPath: __filename,
|
|
30
49
|
args: process.argv.slice(2),
|
|
@@ -4983,8 +5002,10 @@ async function main() {
|
|
|
4983
5002
|
}
|
|
4984
5003
|
}
|
|
4985
5004
|
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
5005
|
+
if (isCliEntrypoint()) {
|
|
5006
|
+
Promise.resolve(main()).catch((err) => {
|
|
5007
|
+
closeReadline();
|
|
5008
|
+
console.error("Setup failed:", err.message);
|
|
5009
|
+
process.exit(1);
|
|
5010
|
+
});
|
|
5011
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.15.0",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/agent_runner.py
CHANGED
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import os
|
|
7
|
+
import re
|
|
7
8
|
import paths
|
|
8
9
|
import shlex
|
|
9
10
|
import shutil
|
|
@@ -385,6 +386,79 @@ def _headless_env(env: dict | None = None) -> dict:
|
|
|
385
386
|
return merged
|
|
386
387
|
|
|
387
388
|
|
|
389
|
+
_MUTATING_TOOL_NAMES = frozenset({
|
|
390
|
+
"write",
|
|
391
|
+
"edit",
|
|
392
|
+
"multiedit",
|
|
393
|
+
"notebookedit",
|
|
394
|
+
"delete",
|
|
395
|
+
"bash",
|
|
396
|
+
"shell",
|
|
397
|
+
})
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _runner_mutating_tools_allowed(allowed_tools: str) -> bool:
|
|
401
|
+
text = str(allowed_tools or "").strip().lower()
|
|
402
|
+
if not text:
|
|
403
|
+
return True
|
|
404
|
+
parts = {part.strip().split(":", 1)[0].lower() for part in re.split(r"[,;\s]+", text) if part.strip()}
|
|
405
|
+
return bool(parts & _MUTATING_TOOL_NAMES)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _extract_runner_guard_paths(prompt: str, cwd: Path) -> list[str]:
|
|
409
|
+
found: set[str] = set()
|
|
410
|
+
text = str(prompt or "")
|
|
411
|
+
for match in re.findall(r"(?<![A-Za-z0-9_])(?:/[^\s'\"`<>]+|[A-Za-z]:\\[^\s'\"`<>]+)", text):
|
|
412
|
+
cleaned = match.rstrip(".,);:]")
|
|
413
|
+
if cleaned:
|
|
414
|
+
found.add(cleaned)
|
|
415
|
+
for match in re.findall(r"(?<![A-Za-z0-9_])(?:src|scripts|tests|docs|lib|renderer|app)/[A-Za-z0-9_./-]+\.[A-Za-z0-9]+", text):
|
|
416
|
+
found.add(str((cwd / match.rstrip(".,);:]")).resolve()))
|
|
417
|
+
try:
|
|
418
|
+
resolved_cwd = cwd.resolve()
|
|
419
|
+
except Exception:
|
|
420
|
+
resolved_cwd = cwd
|
|
421
|
+
runtime_core = NEXO_HOME / "core"
|
|
422
|
+
try:
|
|
423
|
+
if resolved_cwd == runtime_core or runtime_core in resolved_cwd.parents:
|
|
424
|
+
found.add(str(resolved_cwd))
|
|
425
|
+
except Exception:
|
|
426
|
+
pass
|
|
427
|
+
return sorted(found)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _run_headless_runner_guard(*, caller: str, cwd: Path, prompt: str, allowed_tools: str) -> dict:
|
|
431
|
+
if not _runner_mutating_tools_allowed(allowed_tools):
|
|
432
|
+
return {"blocked": False, "skipped": "read_only_tools"}
|
|
433
|
+
guard_paths = _extract_runner_guard_paths(prompt, cwd)
|
|
434
|
+
if not guard_paths:
|
|
435
|
+
return {"blocked": False, "skipped": "no_explicit_paths"}
|
|
436
|
+
try:
|
|
437
|
+
runtime_root = str(NEXO_HOME)
|
|
438
|
+
if runtime_root and runtime_root not in sys.path:
|
|
439
|
+
sys.path.insert(0, runtime_root)
|
|
440
|
+
from plugins.guard import handle_guard_check # type: ignore
|
|
441
|
+
|
|
442
|
+
output = handle_guard_check(
|
|
443
|
+
files=",".join(guard_paths),
|
|
444
|
+
area=f"runner:{caller or 'headless'}",
|
|
445
|
+
project_hint=f"headless runner caller={caller or 'unknown'} cwd={cwd}",
|
|
446
|
+
include_schemas="true",
|
|
447
|
+
)
|
|
448
|
+
except Exception as exc:
|
|
449
|
+
return {
|
|
450
|
+
"blocked": True,
|
|
451
|
+
"summary": f"Runner guard unavailable: {exc}",
|
|
452
|
+
"paths": guard_paths,
|
|
453
|
+
}
|
|
454
|
+
blocked = "BLOCKING RULES" in str(output or "")
|
|
455
|
+
return {
|
|
456
|
+
"blocked": blocked,
|
|
457
|
+
"summary": str(output or ""),
|
|
458
|
+
"paths": guard_paths,
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
|
|
388
462
|
def _load_client_bootstrap_prompt(client: str) -> str:
|
|
389
463
|
try:
|
|
390
464
|
from bootstrap_docs import load_bootstrap_prompt
|
|
@@ -1000,6 +1074,18 @@ def run_automation_prompt(
|
|
|
1000
1074
|
reasoning_effort=reasoning_effort,
|
|
1001
1075
|
preferences=prefs,
|
|
1002
1076
|
)
|
|
1077
|
+
guard_result = _run_headless_runner_guard(
|
|
1078
|
+
caller=caller,
|
|
1079
|
+
cwd=cwd_path,
|
|
1080
|
+
prompt=prompt,
|
|
1081
|
+
allowed_tools=allowed_tools,
|
|
1082
|
+
)
|
|
1083
|
+
if guard_result.get("blocked"):
|
|
1084
|
+
stderr = "NEXO runner guard blocked this automation before editing shared files.\n"
|
|
1085
|
+
summary = str(guard_result.get("summary") or "").strip()
|
|
1086
|
+
if summary:
|
|
1087
|
+
stderr = _append_stderr(stderr, summary)
|
|
1088
|
+
return subprocess.CompletedProcess(["nexo-runner-guard"], 2, "", stderr)
|
|
1003
1089
|
started_at = time.perf_counter()
|
|
1004
1090
|
|
|
1005
1091
|
if selected_backend == CLIENT_CLAUDE_CODE:
|
package/src/claim_graph.py
CHANGED
|
@@ -22,13 +22,28 @@ def _get_db():
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def _embed(text: str) -> np.ndarray:
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
try:
|
|
26
|
+
import cognitive
|
|
27
|
+
return cognitive.embed(text)
|
|
28
|
+
except Exception:
|
|
29
|
+
try:
|
|
30
|
+
import cognitive
|
|
31
|
+
dim = int(getattr(cognitive, "EMBEDDING_DIM", 384) or 384)
|
|
32
|
+
except Exception:
|
|
33
|
+
dim = 768
|
|
34
|
+
return np.zeros(dim, dtype=np.float32)
|
|
27
35
|
|
|
28
36
|
|
|
29
37
|
def _cosine_similarity(a, b) -> float:
|
|
30
|
-
|
|
31
|
-
|
|
38
|
+
try:
|
|
39
|
+
import cognitive
|
|
40
|
+
return cognitive.cosine_similarity(a, b)
|
|
41
|
+
except Exception:
|
|
42
|
+
norm_a = np.linalg.norm(a)
|
|
43
|
+
norm_b = np.linalg.norm(b)
|
|
44
|
+
if norm_a == 0 or norm_b == 0:
|
|
45
|
+
return 0.0
|
|
46
|
+
return float(np.dot(a, b) / (norm_a * norm_b))
|
|
32
47
|
|
|
33
48
|
|
|
34
49
|
def _array_to_blob(arr: np.ndarray) -> bytes:
|
package/src/cognitive/_core.py
CHANGED
|
@@ -3,8 +3,10 @@
|
|
|
3
3
|
import base64
|
|
4
4
|
import json
|
|
5
5
|
import math
|
|
6
|
+
import hashlib
|
|
6
7
|
import os
|
|
7
8
|
import re
|
|
9
|
+
import shutil
|
|
8
10
|
import sqlite3
|
|
9
11
|
import numpy as np
|
|
10
12
|
from datetime import datetime, timedelta
|
|
@@ -18,7 +20,19 @@ _cognitive_dir = paths.cognitive_dir()
|
|
|
18
20
|
_cognitive_dir.mkdir(parents=True, exist_ok=True)
|
|
19
21
|
|
|
20
22
|
COGNITIVE_DB = str(_cognitive_dir / "cognitive.db")
|
|
21
|
-
|
|
23
|
+
def _configured_embedding_dim() -> int:
|
|
24
|
+
try:
|
|
25
|
+
from local_models import get_local_model_spec
|
|
26
|
+
|
|
27
|
+
dim = int(get_local_model_spec("bge-base-embeddings").dimension or 0)
|
|
28
|
+
if dim > 0:
|
|
29
|
+
return dim
|
|
30
|
+
except Exception:
|
|
31
|
+
pass
|
|
32
|
+
return 384
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
EMBEDDING_DIM = _configured_embedding_dim()
|
|
22
36
|
LAMBDA_STM = 0.004126 # half-life = ln(2) / (7 * 24) ≈ 7 days
|
|
23
37
|
LAMBDA_LTM = 0.000481 # half-life = ln(2) / (60 * 24) ≈ 60 days
|
|
24
38
|
DEFAULT_MEMORY_STABILITY = 1.0
|
|
@@ -307,20 +321,37 @@ def _migrate_memory_personalization(conn: sqlite3.Connection):
|
|
|
307
321
|
|
|
308
322
|
|
|
309
323
|
def _auto_migrate_embeddings(conn: sqlite3.Connection):
|
|
310
|
-
"""
|
|
324
|
+
"""Re-embed when vector dimension or pinned embedding model changes."""
|
|
311
325
|
try:
|
|
312
|
-
|
|
326
|
+
conn.execute("""
|
|
327
|
+
CREATE TABLE IF NOT EXISTS embedding_model_state (
|
|
328
|
+
key TEXT PRIMARY KEY,
|
|
329
|
+
value TEXT NOT NULL,
|
|
330
|
+
updated_at TEXT DEFAULT (datetime('now'))
|
|
331
|
+
)
|
|
332
|
+
""")
|
|
333
|
+
current_marker = _current_embedding_model_marker()
|
|
334
|
+
stored = conn.execute(
|
|
335
|
+
"SELECT value FROM embedding_model_state WHERE key = 'embedding_model_marker'"
|
|
336
|
+
).fetchone()
|
|
337
|
+
stored_marker = stored["value"] if stored else ""
|
|
338
|
+
|
|
339
|
+
row = None
|
|
340
|
+
for table in ("stm_memories", "ltm_memories", "quarantine"):
|
|
341
|
+
row = conn.execute(f"SELECT embedding FROM {table} LIMIT 1").fetchone()
|
|
342
|
+
if row:
|
|
343
|
+
break
|
|
313
344
|
if not row:
|
|
314
|
-
|
|
345
|
+
_write_embedding_model_marker(conn, current_marker)
|
|
346
|
+
return
|
|
315
347
|
|
|
316
348
|
vec = np.frombuffer(row["embedding"], dtype=np.float32)
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
return # Unknown dimension, don't touch
|
|
349
|
+
dimension_matches = len(vec) == EMBEDDING_DIM
|
|
350
|
+
model_matches = stored_marker == current_marker
|
|
351
|
+
if dimension_matches and model_matches:
|
|
352
|
+
return
|
|
322
353
|
|
|
323
|
-
|
|
354
|
+
_backup_cognitive_db_for_embedding_migration(stored_marker, current_marker)
|
|
324
355
|
model = _get_model()
|
|
325
356
|
|
|
326
357
|
for table in ("stm_memories", "ltm_memories", "quarantine"):
|
|
@@ -333,14 +364,75 @@ def _auto_migrate_embeddings(conn: sqlite3.Connection):
|
|
|
333
364
|
|
|
334
365
|
embeddings = list(model.embed(contents))
|
|
335
366
|
for mem_id, emb in zip(ids, embeddings):
|
|
336
|
-
|
|
367
|
+
arr = np.array(emb, dtype=np.float32)
|
|
368
|
+
if len(arr) != EMBEDDING_DIM:
|
|
369
|
+
raise ValueError(f"embedding dimension mismatch: {len(arr)} != {EMBEDDING_DIM}")
|
|
370
|
+
blob = arr.tobytes()
|
|
337
371
|
conn.execute(f"UPDATE {table} SET embedding = ? WHERE id = ?", (blob, mem_id))
|
|
338
372
|
|
|
373
|
+
_write_embedding_model_marker(conn, current_marker)
|
|
339
374
|
conn.commit()
|
|
340
375
|
except Exception:
|
|
341
376
|
pass # Don't break startup if migration fails
|
|
342
377
|
|
|
343
378
|
|
|
379
|
+
def _current_embedding_model_marker() -> str:
|
|
380
|
+
try:
|
|
381
|
+
from local_models import get_local_model_spec
|
|
382
|
+
|
|
383
|
+
spec = get_local_model_spec("bge-base-embeddings")
|
|
384
|
+
return "|".join([
|
|
385
|
+
spec.name,
|
|
386
|
+
spec.kind,
|
|
387
|
+
spec.model_id,
|
|
388
|
+
spec.source_repo,
|
|
389
|
+
spec.revision,
|
|
390
|
+
str(EMBEDDING_DIM),
|
|
391
|
+
])
|
|
392
|
+
except Exception:
|
|
393
|
+
return f"unknown|{EMBEDDING_DIM}"
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _write_embedding_model_marker(conn: sqlite3.Connection, marker: str) -> None:
|
|
397
|
+
conn.execute(
|
|
398
|
+
"""
|
|
399
|
+
INSERT INTO embedding_model_state (key, value, updated_at)
|
|
400
|
+
VALUES ('embedding_model_marker', ?, datetime('now'))
|
|
401
|
+
ON CONFLICT(key) DO UPDATE SET
|
|
402
|
+
value = excluded.value,
|
|
403
|
+
updated_at = excluded.updated_at
|
|
404
|
+
""",
|
|
405
|
+
(marker,),
|
|
406
|
+
)
|
|
407
|
+
conn.commit()
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _backup_cognitive_db_for_embedding_migration(old_marker: str, new_marker: str) -> None:
|
|
411
|
+
db_path = Path(COGNITIVE_DB)
|
|
412
|
+
if not db_path.exists():
|
|
413
|
+
return
|
|
414
|
+
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
415
|
+
backup = db_path.with_name(f"{db_path.name}.bak-embedding-{stamp}")
|
|
416
|
+
meta = backup.with_suffix(backup.suffix + ".json")
|
|
417
|
+
try:
|
|
418
|
+
shutil.copy2(db_path, backup)
|
|
419
|
+
meta.write_text(
|
|
420
|
+
json.dumps(
|
|
421
|
+
{
|
|
422
|
+
"old_marker": old_marker,
|
|
423
|
+
"new_marker": new_marker,
|
|
424
|
+
"created_at": datetime.now().isoformat(timespec="seconds"),
|
|
425
|
+
},
|
|
426
|
+
indent=2,
|
|
427
|
+
ensure_ascii=True,
|
|
428
|
+
sort_keys=True,
|
|
429
|
+
) + "\n",
|
|
430
|
+
encoding="utf-8",
|
|
431
|
+
)
|
|
432
|
+
except Exception:
|
|
433
|
+
pass
|
|
434
|
+
|
|
435
|
+
|
|
344
436
|
def _init_tables(conn: sqlite3.Connection):
|
|
345
437
|
"""Create tables if they don't exist."""
|
|
346
438
|
conn.executescript("""
|
|
@@ -558,6 +650,8 @@ def _get_model():
|
|
|
558
650
|
"""Lazy-load fastembed TextEmbedding model."""
|
|
559
651
|
global _model
|
|
560
652
|
if _model is None:
|
|
653
|
+
if _model_download_disabled():
|
|
654
|
+
raise RuntimeError("cognitive model loading disabled for this environment")
|
|
561
655
|
from local_models import build_fastembed_embedding
|
|
562
656
|
|
|
563
657
|
_model = build_fastembed_embedding("bge-base-embeddings")
|
|
@@ -577,6 +671,22 @@ def _get_reranker():
|
|
|
577
671
|
return _reranker if _reranker is not False else None
|
|
578
672
|
|
|
579
673
|
|
|
674
|
+
def _model_download_disabled() -> bool:
|
|
675
|
+
return os.environ.get("NEXO_SKIP_COGNITIVE_MODEL_DOWNLOAD", "").strip().lower() in {"1", "true", "yes"}
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _deterministic_fallback_embedding(text: str) -> np.ndarray:
|
|
679
|
+
"""Return a stable vector for tests/offline fallback paths."""
|
|
680
|
+
digest = hashlib.sha256(str(text or "").encode("utf-8", errors="ignore")).digest()
|
|
681
|
+
arr = np.zeros(EMBEDDING_DIM, dtype=np.float32)
|
|
682
|
+
for index, byte in enumerate(digest):
|
|
683
|
+
arr[index] = (float(byte) / 255.0) - 0.5
|
|
684
|
+
norm = np.linalg.norm(arr)
|
|
685
|
+
if norm > 0:
|
|
686
|
+
arr = arr / norm
|
|
687
|
+
return arr.astype(np.float32)
|
|
688
|
+
|
|
689
|
+
|
|
580
690
|
def rerank_results(query: str, results: list[dict], top_k: int = 5) -> list[dict]:
|
|
581
691
|
"""Rerank search results using cross-encoder for precise top-k.
|
|
582
692
|
|
|
@@ -603,9 +713,11 @@ def rerank_results(query: str, results: list[dict], top_k: int = 5) -> list[dict
|
|
|
603
713
|
|
|
604
714
|
|
|
605
715
|
def embed(text: str) -> np.ndarray:
|
|
606
|
-
"""Embed text into a
|
|
716
|
+
"""Embed text into a float32 vector. Returns zeros for empty text."""
|
|
607
717
|
if not text or not text.strip():
|
|
608
718
|
return np.zeros(EMBEDDING_DIM, dtype=np.float32)
|
|
719
|
+
if _model_download_disabled():
|
|
720
|
+
return _deterministic_fallback_embedding(text)
|
|
609
721
|
model = _get_model()
|
|
610
722
|
embeddings = list(model.embed([text]))
|
|
611
723
|
return np.array(embeddings[0], dtype=np.float32)
|
package/src/cognitive/_search.py
CHANGED
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
import re
|
|
5
5
|
import sqlite3
|
|
6
6
|
import time
|
|
7
|
+
import unicodedata
|
|
7
8
|
import numpy as np
|
|
8
9
|
from datetime import datetime, timezone
|
|
9
10
|
|
|
@@ -357,6 +358,145 @@ def _result_confidence(score: float) -> str:
|
|
|
357
358
|
return "low"
|
|
358
359
|
|
|
359
360
|
|
|
361
|
+
_LEARNING_CONTEXT_MIN_TERM_CHARS = 3
|
|
362
|
+
_LEARNING_CONTEXT_STOP_TERMS = frozenset({
|
|
363
|
+
"all", "any", "the", "and", "for", "from", "with", "sin", "con", "para",
|
|
364
|
+
"por", "que", "del", "las", "los", "una", "uno", "learning", "learn",
|
|
365
|
+
})
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _context_norm(text: object) -> str:
|
|
369
|
+
value = unicodedata.normalize("NFKD", str(text or "").lower())
|
|
370
|
+
value = "".join(ch for ch in value if not unicodedata.combining(ch))
|
|
371
|
+
return re.sub(r"\s+", " ", value.replace("_", " ").strip())
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _context_tokens(text: object) -> set[str]:
|
|
375
|
+
tokens = set()
|
|
376
|
+
for token in re.findall(r"[a-z0-9][a-z0-9.-]*", _context_norm(text)):
|
|
377
|
+
token = token.strip(".-")
|
|
378
|
+
if len(token) >= _LEARNING_CONTEXT_MIN_TERM_CHARS and token not in _LEARNING_CONTEXT_STOP_TERMS:
|
|
379
|
+
tokens.add(token)
|
|
380
|
+
if "-" in token:
|
|
381
|
+
tokens.add(token.replace("-", " "))
|
|
382
|
+
tokens.update(part for part in token.split("-") if len(part) >= _LEARNING_CONTEXT_MIN_TERM_CHARS)
|
|
383
|
+
return tokens
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _learning_context_terms(row: dict) -> set[str]:
|
|
387
|
+
terms: set[str] = set()
|
|
388
|
+
for field in ("domain", "tags"):
|
|
389
|
+
raw = row.get(field, "")
|
|
390
|
+
for chunk in re.split(r"[,;#\n]+", str(raw or "")):
|
|
391
|
+
normalized = _context_norm(chunk)
|
|
392
|
+
if len(normalized) >= _LEARNING_CONTEXT_MIN_TERM_CHARS and normalized not in _LEARNING_CONTEXT_STOP_TERMS:
|
|
393
|
+
terms.add(normalized)
|
|
394
|
+
if "-" in normalized:
|
|
395
|
+
terms.add(normalized.replace("-", " "))
|
|
396
|
+
terms.update(_context_tokens(chunk))
|
|
397
|
+
source_id = _context_norm(row.get("source_id", ""))
|
|
398
|
+
if re.fullmatch(r"l\d+", source_id):
|
|
399
|
+
terms.add(source_id)
|
|
400
|
+
return terms
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _learning_context_match(query_text: str, row: dict) -> str:
|
|
404
|
+
query_norm = _context_norm(query_text)
|
|
405
|
+
if not query_norm:
|
|
406
|
+
return ""
|
|
407
|
+
query_padded = f" {query_norm} "
|
|
408
|
+
query_tokens = _context_tokens(query_norm)
|
|
409
|
+
for term in sorted(_learning_context_terms(row), key=len, reverse=True):
|
|
410
|
+
if " " in term or "-" in term or "." in term:
|
|
411
|
+
variants = {term, term.replace("-", " "), term.replace(" ", "-")}
|
|
412
|
+
if any(f" {variant} " in query_padded for variant in variants):
|
|
413
|
+
return term
|
|
414
|
+
continue
|
|
415
|
+
if term in query_tokens:
|
|
416
|
+
return term
|
|
417
|
+
return ""
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _learning_floor_entry(store: str, row: dict, matched_term: str) -> dict:
|
|
421
|
+
strength = max(1.0, float(row.get("strength") or 0.0))
|
|
422
|
+
entry = {
|
|
423
|
+
"store": store,
|
|
424
|
+
"id": row["id"],
|
|
425
|
+
"content": row["content"],
|
|
426
|
+
"source_type": row["source_type"],
|
|
427
|
+
"source_id": row["source_id"],
|
|
428
|
+
"source_title": row["source_title"],
|
|
429
|
+
"domain": row["domain"],
|
|
430
|
+
"created_at": row["created_at"],
|
|
431
|
+
"strength": strength,
|
|
432
|
+
"access_count": row["access_count"],
|
|
433
|
+
"score": 1.0,
|
|
434
|
+
"learning_context_floor": True,
|
|
435
|
+
"learning_context_match": matched_term,
|
|
436
|
+
"lifecycle_state": row.get("lifecycle_state") or "active",
|
|
437
|
+
}
|
|
438
|
+
if store == "ltm":
|
|
439
|
+
entry["tags"] = row.get("tags", "")
|
|
440
|
+
return entry
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _learning_context_floor_results(
|
|
444
|
+
query_text: str,
|
|
445
|
+
*,
|
|
446
|
+
stores: str = "both",
|
|
447
|
+
source_type_filter: str = "",
|
|
448
|
+
include_archived: bool = False,
|
|
449
|
+
top_k: int = 10,
|
|
450
|
+
) -> list[dict]:
|
|
451
|
+
if source_type_filter and source_type_filter != "learning":
|
|
452
|
+
return []
|
|
453
|
+
db = _get_db()
|
|
454
|
+
results: list[dict] = []
|
|
455
|
+
store_specs = []
|
|
456
|
+
if stores in ("both", "stm"):
|
|
457
|
+
store_specs.append(("stm", "stm_memories", "promoted_to_ltm = 0"))
|
|
458
|
+
if stores in ("both", "ltm"):
|
|
459
|
+
store_specs.append(("ltm", "ltm_memories", "is_dormant = 0"))
|
|
460
|
+
lifecycle = "AND (lifecycle_state IS NULL OR lifecycle_state = 'active' OR lifecycle_state = 'pinned'"
|
|
461
|
+
if include_archived:
|
|
462
|
+
lifecycle += " OR lifecycle_state = 'archived'"
|
|
463
|
+
lifecycle += ")"
|
|
464
|
+
for store, table, base_where in store_specs:
|
|
465
|
+
rows = db.execute(
|
|
466
|
+
f"SELECT * FROM {table} WHERE {base_where} AND source_type = 'learning' {lifecycle} "
|
|
467
|
+
"ORDER BY strength DESC, access_count DESC, created_at DESC LIMIT ?",
|
|
468
|
+
(max(10, int(top_k) * 8),),
|
|
469
|
+
).fetchall()
|
|
470
|
+
for row in rows:
|
|
471
|
+
row_dict = dict(row)
|
|
472
|
+
matched = _learning_context_match(query_text, row_dict)
|
|
473
|
+
if matched:
|
|
474
|
+
results.append(_learning_floor_entry(store, row_dict, matched))
|
|
475
|
+
results.sort(key=lambda r: (r.get("strength", 0), r.get("access_count", 0), r.get("created_at", "")), reverse=True)
|
|
476
|
+
return results[: max(1, int(top_k))]
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _merge_learning_context_floor(results: list[dict], floor_results: list[dict], top_k: int | None = None) -> list[dict]:
|
|
480
|
+
if not floor_results:
|
|
481
|
+
return results
|
|
482
|
+
by_key = {(r.get("store"), r.get("id")): r for r in results}
|
|
483
|
+
for floor in floor_results:
|
|
484
|
+
key = (floor.get("store"), floor.get("id"))
|
|
485
|
+
current = by_key.get(key)
|
|
486
|
+
if current is None:
|
|
487
|
+
results.append(floor.copy())
|
|
488
|
+
by_key[key] = results[-1]
|
|
489
|
+
continue
|
|
490
|
+
current["score"] = max(float(current.get("score") or 0.0), 1.0)
|
|
491
|
+
current["strength"] = max(float(current.get("strength") or 0.0), 1.0)
|
|
492
|
+
current["learning_context_floor"] = True
|
|
493
|
+
current["learning_context_match"] = floor.get("learning_context_match", "")
|
|
494
|
+
results.sort(key=lambda x: x.get("score", 0), reverse=True)
|
|
495
|
+
if top_k is not None:
|
|
496
|
+
return results[: max(1, int(top_k))]
|
|
497
|
+
return results
|
|
498
|
+
|
|
499
|
+
|
|
360
500
|
# ============================================================================
|
|
361
501
|
# FEATURE 0.5: Knowledge Graph Boost
|
|
362
502
|
# Memories connected to more KG nodes (files, areas, other learnings) are
|
|
@@ -1146,6 +1286,17 @@ def search(
|
|
|
1146
1286
|
# warnings about painful areas more aggressively.
|
|
1147
1287
|
results = _somatic_boost_results(results)
|
|
1148
1288
|
|
|
1289
|
+
# Deterministic learning floor: category/tag/source-id matches must surface
|
|
1290
|
+
# even when semantic similarity alone would miss the operational context.
|
|
1291
|
+
learning_floor_results = _learning_context_floor_results(
|
|
1292
|
+
query_text,
|
|
1293
|
+
stores=stores,
|
|
1294
|
+
source_type_filter=source_type_filter,
|
|
1295
|
+
include_archived=include_archived,
|
|
1296
|
+
top_k=top_k,
|
|
1297
|
+
)
|
|
1298
|
+
results = _merge_learning_context_floor(results, learning_floor_results)
|
|
1299
|
+
|
|
1149
1300
|
# Sort by score descending, take top-20 for reranking
|
|
1150
1301
|
results.sort(key=lambda x: x.get("score", 0), reverse=True)
|
|
1151
1302
|
|
|
@@ -1154,6 +1305,7 @@ def search(
|
|
|
1154
1305
|
results = rerank_results(query_text, results[:top_k * 4], top_k=top_k)
|
|
1155
1306
|
else:
|
|
1156
1307
|
results = results[:top_k]
|
|
1308
|
+
results = _merge_learning_context_floor(results, learning_floor_results, top_k=top_k)
|
|
1157
1309
|
|
|
1158
1310
|
# Spreading activation: boost co-activated neighbors (Feature 2)
|
|
1159
1311
|
co_activation_applied = False
|
|
@@ -1213,6 +1365,8 @@ def search(
|
|
|
1213
1365
|
results.sort(key=lambda x: x["score"], reverse=True)
|
|
1214
1366
|
results = results[:top_k]
|
|
1215
1367
|
|
|
1368
|
+
results = _merge_learning_context_floor(results, learning_floor_results, top_k=top_k)
|
|
1369
|
+
|
|
1216
1370
|
# Add rank explanations
|
|
1217
1371
|
for rank, r in enumerate(results, 1):
|
|
1218
1372
|
score = r["score"]
|
|
@@ -1233,6 +1387,8 @@ def search(
|
|
|
1233
1387
|
parts.append(f"kg_boost=+{r['kg_boost']:.3f} ({r.get('kg_connections', 0)} edges)")
|
|
1234
1388
|
if r.get("co_activation_boost"):
|
|
1235
1389
|
parts.append(f"co_activation_boost=+{r['co_activation_boost']:.3f}")
|
|
1390
|
+
if r.get("learning_context_floor"):
|
|
1391
|
+
parts.append(f"learning_context_floor=1.000 match={r.get('learning_context_match', '')}")
|
|
1236
1392
|
if use_hyde is None and resolved_use_hyde:
|
|
1237
1393
|
parts.append("hyde=auto")
|
|
1238
1394
|
if spreading_depth is None and resolved_spreading_depth > 0:
|
package/src/db/_learnings.py
CHANGED
|
@@ -146,18 +146,30 @@ def search_learnings(query: str, category: str = None) -> list[dict]:
|
|
|
146
146
|
return [dict(r) for r in rows]
|
|
147
147
|
|
|
148
148
|
|
|
149
|
-
def list_learnings(
|
|
150
|
-
|
|
149
|
+
def list_learnings(
|
|
150
|
+
category: str = None,
|
|
151
|
+
created_after: float | None = None,
|
|
152
|
+
created_before: float | None = None,
|
|
153
|
+
) -> list[dict]:
|
|
154
|
+
"""List all learnings, optionally filtered by category and creation time."""
|
|
151
155
|
conn = _core().get_db()
|
|
156
|
+
where = []
|
|
157
|
+
params: list[object] = []
|
|
152
158
|
if category:
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
159
|
+
where.append("category = ?")
|
|
160
|
+
params.append(category)
|
|
161
|
+
if created_after is not None:
|
|
162
|
+
where.append("created_at >= ?")
|
|
163
|
+
params.append(float(created_after))
|
|
164
|
+
if created_before is not None:
|
|
165
|
+
where.append("created_at <= ?")
|
|
166
|
+
params.append(float(created_before))
|
|
167
|
+
where_sql = f"WHERE {' AND '.join(where)}" if where else ""
|
|
168
|
+
order_sql = "updated_at DESC" if category else "category ASC, updated_at DESC"
|
|
169
|
+
rows = conn.execute(
|
|
170
|
+
f"SELECT * FROM learnings {where_sql} ORDER BY {order_sql}",
|
|
171
|
+
tuple(params),
|
|
172
|
+
).fetchall()
|
|
161
173
|
return [dict(r) for r in rows]
|
|
162
174
|
|
|
163
175
|
|