ltcai 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -24
- package/docs/CHANGELOG.md +84 -0
- package/docs/V4_2_BRAIN_CORE_ARCHITECTURE.md +97 -0
- package/docs/V4_2_STORAGE_MIGRATION_REPORT.md +91 -0
- package/docs/V4_2_VALIDATION_REPORT.md +89 -0
- package/docs/V4_3_PORTABILITY_ARCHITECTURE.md +69 -0
- package/docs/V4_3_PRIVACY_AUDIT.md +60 -0
- package/docs/V4_3_PRODUCT_HARDENING_REPORT.md +53 -0
- package/docs/V4_3_VALIDATION_REPORT.md +58 -0
- package/docs/V4_DIGITAL_BRAIN_RECOVERY.md +31 -33
- package/frontend/openapi.json +449 -1
- package/frontend/src/api/client.ts +10 -0
- package/frontend/src/api/openapi.ts +542 -0
- package/frontend/src/pages/System.tsx +92 -0
- package/kg_schema.py +1 -1
- package/knowledge_graph.py +4 -4
- package/lattice_brain/__init__.py +70 -0
- package/lattice_brain/_kg_common.py +1 -0
- package/lattice_brain/archive.py +446 -0
- package/lattice_brain/context.py +3 -0
- package/lattice_brain/conversations.py +3 -0
- package/lattice_brain/core.py +82 -0
- package/lattice_brain/discovery.py +1 -0
- package/lattice_brain/documents.py +1 -0
- package/lattice_brain/embeddings.py +82 -0
- package/lattice_brain/identity.py +13 -0
- package/lattice_brain/ingest.py +1 -0
- package/lattice_brain/memory.py +3 -0
- package/lattice_brain/network.py +1 -0
- package/lattice_brain/projection.py +1 -0
- package/lattice_brain/provenance.py +1 -0
- package/lattice_brain/retrieval.py +1 -0
- package/lattice_brain/schema.py +1 -0
- package/lattice_brain/storage/__init__.py +22 -0
- package/lattice_brain/storage/base.py +72 -0
- package/lattice_brain/storage/docker.py +105 -0
- package/lattice_brain/storage/factory.py +31 -0
- package/lattice_brain/storage/migration.py +190 -0
- package/lattice_brain/storage/postgres.py +123 -0
- package/lattice_brain/storage/sqlite.py +128 -0
- package/lattice_brain/store.py +3 -0
- package/lattice_brain/write_master.py +1 -0
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/admin.py +11 -0
- package/latticeai/api/portability.py +127 -1
- package/latticeai/app_factory.py +26 -10
- package/latticeai/brain/__init__.py +6 -6
- package/latticeai/brain/_kg_common.py +1 -1
- package/latticeai/brain/network.py +1 -1
- package/latticeai/brain/retrieval.py +15 -0
- package/latticeai/brain/store.py +22 -6
- package/latticeai/core/config.py +9 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/multi_agent.py +1 -1
- package/latticeai/core/product_hardening.py +217 -0
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/services/kg_portability.py +227 -3
- package/ltcai_cli.py +2 -1
- package/package.json +4 -3
- package/scripts/bump_version.py +3 -0
- package/scripts/clean_release_artifacts.mjs +27 -0
- package/scripts/lint_frontend.mjs +10 -0
- package/scripts/migrate_brain_storage.py +53 -0
- package/scripts/validate_release_artifacts.py +10 -0
- package/scripts/wheel_smoke.py +3 -0
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/src/main.rs +113 -13
- package/src-tauri/tauri.conf.json +5 -2
- package/static/app/asset-manifest.json +5 -5
- package/static/app/assets/{index-CJRAzNnf.js → index-RiJTJliG.js} +3 -3
- package/static/app/assets/index-RiJTJliG.js.map +1 -0
- package/static/app/assets/index-yZswHE3d.css +2 -0
- package/static/app/index.html +2 -2
- package/static/app/assets/index-CJRAzNnf.js.map +0 -1
- package/static/app/assets/index-CSwBBgf4.css +0 -2
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Opt-in Postgres/pgvector storage engine for Brain Core scale mode."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict
|
|
8
|
+
|
|
9
|
+
from .base import StorageCapabilities, StorageEngine, StorageUnavailable
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _quote_ident(value: str) -> str:
|
|
13
|
+
return '"' + str(value).replace('"', '""') + '"'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class PostgresConfig:
|
|
18
|
+
dsn: str
|
|
19
|
+
schema: str = "lattice_brain"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PostgresEngine(StorageEngine):
|
|
23
|
+
name = "postgres"
|
|
24
|
+
|
|
25
|
+
def __init__(self, dsn: str, *, schema: str = "lattice_brain") -> None:
|
|
26
|
+
self.config = PostgresConfig(dsn=dsn, schema=schema)
|
|
27
|
+
|
|
28
|
+
def _psycopg(self):
|
|
29
|
+
try:
|
|
30
|
+
import psycopg # type: ignore
|
|
31
|
+
except Exception as exc:
|
|
32
|
+
raise StorageUnavailable(
|
|
33
|
+
"Postgres storage requires optional dependency 'psycopg'. "
|
|
34
|
+
"Install the postgres extra before selecting LATTICEAI_STORAGE_ENGINE=postgres."
|
|
35
|
+
) from exc
|
|
36
|
+
return psycopg
|
|
37
|
+
|
|
38
|
+
def connect(self) -> Any:
|
|
39
|
+
if not self.config.dsn:
|
|
40
|
+
raise StorageUnavailable(
|
|
41
|
+
"Postgres storage requires LATTICEAI_POSTGRES_DSN; no SQLite fallback is attempted."
|
|
42
|
+
)
|
|
43
|
+
psycopg = self._psycopg()
|
|
44
|
+
return psycopg.connect(self.config.dsn)
|
|
45
|
+
|
|
46
|
+
def initialize(self) -> Dict[str, Any]:
|
|
47
|
+
schema = _quote_ident(self.config.schema)
|
|
48
|
+
with self.connect() as conn:
|
|
49
|
+
with conn.cursor() as cur:
|
|
50
|
+
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
51
|
+
cur.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}")
|
|
52
|
+
cur.execute(
|
|
53
|
+
f"""
|
|
54
|
+
CREATE TABLE IF NOT EXISTS {schema}.storage_meta (
|
|
55
|
+
key text PRIMARY KEY,
|
|
56
|
+
value text NOT NULL,
|
|
57
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
58
|
+
)
|
|
59
|
+
"""
|
|
60
|
+
)
|
|
61
|
+
cur.execute(
|
|
62
|
+
f"""
|
|
63
|
+
CREATE TABLE IF NOT EXISTS {schema}.brain_vectors (
|
|
64
|
+
item_id text PRIMARY KEY,
|
|
65
|
+
item_type text NOT NULL,
|
|
66
|
+
source_node text NOT NULL,
|
|
67
|
+
text_hash text NOT NULL,
|
|
68
|
+
embedding vector,
|
|
69
|
+
embedding_dim integer NOT NULL,
|
|
70
|
+
embedding_model text NOT NULL,
|
|
71
|
+
metadata_json jsonb NOT NULL DEFAULT '{{}}'::jsonb,
|
|
72
|
+
indexed_at timestamptz NOT NULL DEFAULT now()
|
|
73
|
+
)
|
|
74
|
+
"""
|
|
75
|
+
)
|
|
76
|
+
cur.execute(
|
|
77
|
+
f"""
|
|
78
|
+
INSERT INTO {schema}.storage_meta(key, value)
|
|
79
|
+
VALUES ('engine', 'postgres')
|
|
80
|
+
ON CONFLICT (key) DO UPDATE
|
|
81
|
+
SET value = EXCLUDED.value, updated_at = now()
|
|
82
|
+
"""
|
|
83
|
+
)
|
|
84
|
+
return {"engine": self.name, "schema": self.config.schema}
|
|
85
|
+
|
|
86
|
+
def capabilities(self) -> StorageCapabilities:
|
|
87
|
+
try:
|
|
88
|
+
with self.connect() as conn:
|
|
89
|
+
with conn.cursor() as cur:
|
|
90
|
+
cur.execute("SELECT extname FROM pg_extension WHERE extname='vector'")
|
|
91
|
+
pgvector = cur.fetchone() is not None
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
return StorageCapabilities(
|
|
94
|
+
engine=self.name,
|
|
95
|
+
available=False,
|
|
96
|
+
reason=str(exc),
|
|
97
|
+
vector_backend="pgvector",
|
|
98
|
+
metadata={"schema": self.config.schema},
|
|
99
|
+
)
|
|
100
|
+
return StorageCapabilities(
|
|
101
|
+
engine=self.name,
|
|
102
|
+
available=True,
|
|
103
|
+
reason=None if pgvector else "pgvector extension is not installed",
|
|
104
|
+
vector_backend="pgvector",
|
|
105
|
+
vector_available=pgvector,
|
|
106
|
+
backup_restore=False,
|
|
107
|
+
migrations=True,
|
|
108
|
+
encrypted_archives=False,
|
|
109
|
+
metadata={"schema": self.config.schema},
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
def backup(self, destination: Path) -> Dict[str, Any]:
|
|
113
|
+
raise StorageUnavailable(
|
|
114
|
+
"Postgres logical backup is not implemented inside the app; use pg_dump for this engine."
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def restore(self, source: Path) -> Dict[str, Any]:
|
|
118
|
+
raise StorageUnavailable(
|
|
119
|
+
"Postgres restore is not implemented inside the app; use pg_restore/psql for this engine."
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
__all__ = ["PostgresConfig", "PostgresEngine", "_quote_ident"]
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""SQLite storage engine for Brain Core.
|
|
2
|
+
|
|
3
|
+
SQLite is the default and remains fully local-first. sqlite-vec is detected and
|
|
4
|
+
loaded when present; otherwise the existing brute-force cosine path remains the
|
|
5
|
+
honest, real fallback and is surfaced in capability reports.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import shutil
|
|
11
|
+
import sqlite3
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Dict, Optional
|
|
14
|
+
|
|
15
|
+
from .base import StorageCapabilities, StorageEngine
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _load_sqlite_vec(conn: sqlite3.Connection) -> tuple[bool, Optional[str]]:
|
|
19
|
+
try:
|
|
20
|
+
import sqlite_vec # type: ignore
|
|
21
|
+
except Exception as exc:
|
|
22
|
+
return False, f"sqlite-vec Python package not installed: {exc}"
|
|
23
|
+
try:
|
|
24
|
+
conn.enable_load_extension(True)
|
|
25
|
+
except Exception:
|
|
26
|
+
pass
|
|
27
|
+
try:
|
|
28
|
+
sqlite_vec.load(conn)
|
|
29
|
+
except Exception as exc:
|
|
30
|
+
return False, f"sqlite-vec extension failed to load: {exc}"
|
|
31
|
+
return True, None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class SQLiteEngine(StorageEngine):
|
|
35
|
+
name = "sqlite"
|
|
36
|
+
|
|
37
|
+
def __init__(self, db_path: Path, *, load_vec: bool = True) -> None:
|
|
38
|
+
self.db_path = Path(db_path)
|
|
39
|
+
self.load_vec = bool(load_vec)
|
|
40
|
+
self._sqlite_vec_loaded = False
|
|
41
|
+
self._sqlite_vec_reason: Optional[str] = None
|
|
42
|
+
|
|
43
|
+
def connect(self) -> sqlite3.Connection:
|
|
44
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
conn = sqlite3.connect(str(self.db_path))
|
|
46
|
+
conn.row_factory = sqlite3.Row
|
|
47
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
48
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
49
|
+
if self.load_vec:
|
|
50
|
+
loaded, reason = _load_sqlite_vec(conn)
|
|
51
|
+
self._sqlite_vec_loaded = loaded
|
|
52
|
+
self._sqlite_vec_reason = reason
|
|
53
|
+
return conn
|
|
54
|
+
|
|
55
|
+
def initialize(self) -> Dict[str, Any]:
|
|
56
|
+
with self.connect() as conn:
|
|
57
|
+
conn.execute(
|
|
58
|
+
"""
|
|
59
|
+
CREATE TABLE IF NOT EXISTS storage_meta (
|
|
60
|
+
key TEXT PRIMARY KEY,
|
|
61
|
+
value TEXT NOT NULL
|
|
62
|
+
)
|
|
63
|
+
"""
|
|
64
|
+
)
|
|
65
|
+
conn.execute(
|
|
66
|
+
"INSERT OR REPLACE INTO storage_meta(key, value) VALUES ('engine', 'sqlite')"
|
|
67
|
+
)
|
|
68
|
+
return {"engine": self.name, "db_path": str(self.db_path)}
|
|
69
|
+
|
|
70
|
+
def capabilities(self) -> StorageCapabilities:
|
|
71
|
+
if not self.db_path.parent.exists():
|
|
72
|
+
return StorageCapabilities(
|
|
73
|
+
engine=self.name,
|
|
74
|
+
available=True,
|
|
75
|
+
vector_backend="bruteforce-cosine",
|
|
76
|
+
vector_available=True,
|
|
77
|
+
backup_restore=True,
|
|
78
|
+
migrations=True,
|
|
79
|
+
encrypted_archives=True,
|
|
80
|
+
metadata={"db_path": str(self.db_path), "sqlite_vec_loaded": False},
|
|
81
|
+
)
|
|
82
|
+
# Probe on demand so status is accurate even before the graph opens.
|
|
83
|
+
try:
|
|
84
|
+
with self.connect():
|
|
85
|
+
pass
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
return StorageCapabilities(
|
|
88
|
+
engine=self.name,
|
|
89
|
+
available=False,
|
|
90
|
+
reason=str(exc),
|
|
91
|
+
metadata={"db_path": str(self.db_path)},
|
|
92
|
+
)
|
|
93
|
+
vector_backend = "sqlite-vec" if self._sqlite_vec_loaded else "bruteforce-cosine"
|
|
94
|
+
return StorageCapabilities(
|
|
95
|
+
engine=self.name,
|
|
96
|
+
available=True,
|
|
97
|
+
reason=None if self._sqlite_vec_loaded else self._sqlite_vec_reason,
|
|
98
|
+
vector_backend=vector_backend,
|
|
99
|
+
vector_available=True,
|
|
100
|
+
backup_restore=True,
|
|
101
|
+
migrations=True,
|
|
102
|
+
encrypted_archives=True,
|
|
103
|
+
metadata={
|
|
104
|
+
"db_path": str(self.db_path),
|
|
105
|
+
"sqlite_vec_loaded": self._sqlite_vec_loaded,
|
|
106
|
+
},
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def backup(self, destination: Path) -> Dict[str, Any]:
|
|
110
|
+
dest = Path(destination)
|
|
111
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
with self.connect() as src, sqlite3.connect(str(dest)) as dst:
|
|
113
|
+
src.backup(dst)
|
|
114
|
+
return {"engine": self.name, "path": str(dest), "bytes": dest.stat().st_size}
|
|
115
|
+
|
|
116
|
+
def restore(self, source: Path) -> Dict[str, Any]:
|
|
117
|
+
src = Path(source)
|
|
118
|
+
if not src.exists():
|
|
119
|
+
raise FileNotFoundError(f"SQLite backup not found: {src}")
|
|
120
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
121
|
+
for sibling in (self.db_path, Path(str(self.db_path) + "-wal"), Path(str(self.db_path) + "-shm")):
|
|
122
|
+
if sibling.exists():
|
|
123
|
+
sibling.unlink()
|
|
124
|
+
shutil.copyfile(src, self.db_path)
|
|
125
|
+
return {"engine": self.name, "restored": True, "path": str(self.db_path)}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
__all__ = ["SQLiteEngine"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from latticeai.brain.write_master import * # noqa: F401,F403
|
package/latticeai/__init__.py
CHANGED
package/latticeai/api/admin.py
CHANGED
|
@@ -56,6 +56,7 @@ def create_admin_router(
|
|
|
56
56
|
invite_gate_enabled: bool,
|
|
57
57
|
default_port: int,
|
|
58
58
|
policy_matrix: Optional[Callable[[], List[Dict[str, object]]]] = None,
|
|
59
|
+
product_hardening_status: Optional[Callable[[], Dict[str, object]]] = None,
|
|
59
60
|
) -> APIRouter:
|
|
60
61
|
router = APIRouter()
|
|
61
62
|
|
|
@@ -155,6 +156,16 @@ def create_admin_router(
|
|
|
155
156
|
]
|
|
156
157
|
}
|
|
157
158
|
|
|
159
|
+
@router.get("/admin/product-hardening")
|
|
160
|
+
async def admin_product_hardening(request: Request):
|
|
161
|
+
require_admin(request)
|
|
162
|
+
if product_hardening_status is None:
|
|
163
|
+
return {
|
|
164
|
+
"available": False,
|
|
165
|
+
"reason": "Product hardening status provider is not configured.",
|
|
166
|
+
}
|
|
167
|
+
return product_hardening_status()
|
|
168
|
+
|
|
158
169
|
@router.get("/vpc/status")
|
|
159
170
|
async def vpc_status(request: Request):
|
|
160
171
|
require_user(request)
|
|
@@ -26,6 +26,42 @@ class BackupRequest(BaseModel):
|
|
|
26
26
|
class RestoreRequest(BaseModel):
|
|
27
27
|
path: str
|
|
28
28
|
verify: bool = True
|
|
29
|
+
dry_run: bool = False
|
|
30
|
+
confirm: bool = False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class EncryptedArchiveRequest(BaseModel):
|
|
34
|
+
path: Optional[str] = None
|
|
35
|
+
passphrase: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class EncryptedRestoreRequest(BaseModel):
|
|
39
|
+
path: str
|
|
40
|
+
passphrase: str
|
|
41
|
+
dry_run: bool = False
|
|
42
|
+
confirm: bool = False
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class EncryptedInspectRequest(BaseModel):
|
|
46
|
+
path: str
|
|
47
|
+
passphrase: Optional[str] = None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class EncryptedVerifyRequest(BaseModel):
|
|
51
|
+
path: str
|
|
52
|
+
passphrase: str
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DockerPostgresRequest(BaseModel):
|
|
56
|
+
consent: bool = False
|
|
57
|
+
dry_run: bool = False
|
|
58
|
+
port: int = 5432
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SQLiteToPostgresRequest(BaseModel):
|
|
62
|
+
dsn: str
|
|
63
|
+
schema_name: str = "lattice_brain"
|
|
64
|
+
dry_run: bool = True
|
|
29
65
|
|
|
30
66
|
|
|
31
67
|
def create_portability_router(
|
|
@@ -46,6 +82,18 @@ def create_portability_router(
|
|
|
46
82
|
_require_service()
|
|
47
83
|
return service.snapshot_metadata()
|
|
48
84
|
|
|
85
|
+
@router.get("/api/brain/storage")
|
|
86
|
+
async def brain_storage_status(request: Request):
|
|
87
|
+
require_user(request)
|
|
88
|
+
_require_service()
|
|
89
|
+
return service.storage_status()
|
|
90
|
+
|
|
91
|
+
@router.get("/api/knowledge-graph/backup-health")
|
|
92
|
+
async def backup_health(request: Request):
|
|
93
|
+
require_user(request)
|
|
94
|
+
_require_service()
|
|
95
|
+
return service.backup_health()
|
|
96
|
+
|
|
49
97
|
@router.get("/api/knowledge-graph/provenance")
|
|
50
98
|
async def recent_provenance(request: Request, limit: int = 50, source_type: Optional[str] = None):
|
|
51
99
|
"""Recent ingestions (provenance trail) for the ingestion-sources UI."""
|
|
@@ -86,8 +134,86 @@ def create_portability_router(
|
|
|
86
134
|
require_admin(request)
|
|
87
135
|
_require_service()
|
|
88
136
|
try:
|
|
89
|
-
return service.restore(req.path, verify=req.verify)
|
|
137
|
+
return service.restore(req.path, verify=req.verify, dry_run=req.dry_run, confirm=req.confirm)
|
|
90
138
|
except (ValueError, FileNotFoundError) as exc:
|
|
91
139
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
92
140
|
|
|
141
|
+
@router.post("/api/knowledge-graph/archive")
|
|
142
|
+
async def encrypted_archive(req: EncryptedArchiveRequest, request: Request):
|
|
143
|
+
require_admin(request)
|
|
144
|
+
_require_service()
|
|
145
|
+
try:
|
|
146
|
+
return service.encrypted_archive(req.path, passphrase=req.passphrase)
|
|
147
|
+
except (ValueError, FileNotFoundError) as exc:
|
|
148
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
149
|
+
|
|
150
|
+
@router.post("/api/knowledge-graph/archive/inspect")
|
|
151
|
+
async def inspect_encrypted_archive(req: EncryptedInspectRequest, request: Request):
|
|
152
|
+
require_admin(request)
|
|
153
|
+
_require_service()
|
|
154
|
+
try:
|
|
155
|
+
return service.inspect_encrypted_archive(req.path, passphrase=req.passphrase)
|
|
156
|
+
except (ValueError, FileNotFoundError) as exc:
|
|
157
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
158
|
+
|
|
159
|
+
@router.post("/api/knowledge-graph/archive/verify")
|
|
160
|
+
async def verify_encrypted_archive(req: EncryptedVerifyRequest, request: Request):
|
|
161
|
+
require_admin(request)
|
|
162
|
+
_require_service()
|
|
163
|
+
result = service.verify_encrypted_archive(req.path, passphrase=req.passphrase)
|
|
164
|
+
if not result.get("ok"):
|
|
165
|
+
raise HTTPException(status_code=400, detail="; ".join(result.get("errors") or ["Archive verification failed."]))
|
|
166
|
+
return result
|
|
167
|
+
|
|
168
|
+
@router.post("/api/knowledge-graph/archive/import")
|
|
169
|
+
async def import_encrypted_archive(req: EncryptedRestoreRequest, request: Request):
|
|
170
|
+
require_admin(request)
|
|
171
|
+
_require_service()
|
|
172
|
+
try:
|
|
173
|
+
return service.import_encrypted_archive(
|
|
174
|
+
req.path,
|
|
175
|
+
passphrase=req.passphrase,
|
|
176
|
+
dry_run=req.dry_run,
|
|
177
|
+
confirm=req.confirm,
|
|
178
|
+
)
|
|
179
|
+
except (ValueError, FileNotFoundError) as exc:
|
|
180
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
181
|
+
|
|
182
|
+
@router.post("/api/knowledge-graph/archive/restore")
|
|
183
|
+
async def restore_encrypted_archive(req: EncryptedRestoreRequest, request: Request):
|
|
184
|
+
require_admin(request)
|
|
185
|
+
_require_service()
|
|
186
|
+
try:
|
|
187
|
+
return service.restore_encrypted_archive(
|
|
188
|
+
req.path,
|
|
189
|
+
passphrase=req.passphrase,
|
|
190
|
+
dry_run=req.dry_run,
|
|
191
|
+
confirm=req.confirm,
|
|
192
|
+
)
|
|
193
|
+
except (ValueError, FileNotFoundError) as exc:
|
|
194
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
195
|
+
|
|
196
|
+
@router.post("/api/brain/storage/postgres/docker")
|
|
197
|
+
async def setup_postgres_docker(req: DockerPostgresRequest, request: Request):
|
|
198
|
+
require_admin(request)
|
|
199
|
+
_require_service()
|
|
200
|
+
return service.postgres_docker_setup(
|
|
201
|
+
consent=req.consent,
|
|
202
|
+
dry_run=req.dry_run,
|
|
203
|
+
port=req.port,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
@router.post("/api/brain/storage/migrate-postgres")
|
|
207
|
+
async def migrate_sqlite_to_postgres(req: SQLiteToPostgresRequest, request: Request):
|
|
208
|
+
require_admin(request)
|
|
209
|
+
_require_service()
|
|
210
|
+
try:
|
|
211
|
+
return service.migrate_sqlite_to_postgres(
|
|
212
|
+
dsn=req.dsn,
|
|
213
|
+
schema=req.schema_name,
|
|
214
|
+
dry_run=req.dry_run,
|
|
215
|
+
)
|
|
216
|
+
except (ValueError, FileNotFoundError, RuntimeError) as exc:
|
|
217
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
218
|
+
|
|
93
219
|
return router
|
package/latticeai/app_factory.py
CHANGED
|
@@ -60,7 +60,7 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
60
60
|
from pydantic import BaseModel
|
|
61
61
|
|
|
62
62
|
from latticeai.models.router import LLMRouter, normalize_branding
|
|
63
|
-
from knowledge_graph import
|
|
63
|
+
from knowledge_graph import set_llm_router
|
|
64
64
|
from local_knowledge_api import LocalKnowledgeWatcher
|
|
65
65
|
from latticeai.core.security import (
|
|
66
66
|
hash_password,
|
|
@@ -154,6 +154,7 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
154
154
|
from latticeai.api.hooks import create_hooks_router
|
|
155
155
|
from latticeai.core.hooks import HooksRegistry
|
|
156
156
|
from latticeai.core.builtin_hooks import register_builtin_hook_runners
|
|
157
|
+
from latticeai.core.product_hardening import build_product_hardening_status
|
|
157
158
|
from latticeai.api.agent_registry import create_agent_registry_router
|
|
158
159
|
from latticeai.core.agent_registry import AgentRegistry
|
|
159
160
|
from latticeai.api.memory import create_memory_router
|
|
@@ -161,11 +162,12 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
161
162
|
from latticeai.api.portability import create_portability_router
|
|
162
163
|
from latticeai.services.memory_service import MemoryService
|
|
163
164
|
from latticeai.services.ingestion import IngestionItem, IngestionPipeline
|
|
164
|
-
from
|
|
165
|
-
from
|
|
166
|
-
from
|
|
167
|
-
from
|
|
168
|
-
from
|
|
165
|
+
from lattice_brain import BrainCore, ConversationStore
|
|
166
|
+
from lattice_brain.storage import storage_from_env
|
|
167
|
+
from lattice_brain.context import ContextAssembler
|
|
168
|
+
from lattice_brain.memory import BrainMemory
|
|
169
|
+
from lattice_brain.identity import DeviceIdentity
|
|
170
|
+
from lattice_brain.network import BrainNetwork
|
|
169
171
|
from latticeai.api.network import create_network_router
|
|
170
172
|
from latticeai.services.kg_portability import KGPortabilityService
|
|
171
173
|
# The aliased names below look unused but are part of the legacy
|
|
@@ -342,16 +344,22 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
342
344
|
)
|
|
343
345
|
if EMBEDDER.fell_back:
|
|
344
346
|
logging.warning("Embedding provider %s unavailable: %s", EMBEDDER.requested, EMBEDDER.detail)
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
DATA_DIR
|
|
347
|
+
STORAGE_ENGINE = storage_from_env(os.environ, data_dir=DATA_DIR) if ENABLE_GRAPH else None
|
|
348
|
+
BRAIN_CORE = BrainCore.from_paths(
|
|
349
|
+
DATA_DIR,
|
|
348
350
|
embedder=EMBEDDER.provider,
|
|
351
|
+
storage_engine=STORAGE_ENGINE,
|
|
349
352
|
) if ENABLE_GRAPH else None
|
|
353
|
+
KNOWLEDGE_GRAPH = BRAIN_CORE.knowledge if BRAIN_CORE is not None else None
|
|
350
354
|
# ── v4 durable conversation store: unbounded episodic memory in the same
|
|
351
355
|
# SQLite file as the graph (kg_portability backup/restore covers it for
|
|
352
356
|
# free). Legacy chat_history.json is imported once, idempotently, and the
|
|
353
357
|
# file is left untouched on disk as the import source.
|
|
354
|
-
CONVERSATIONS =
|
|
358
|
+
CONVERSATIONS = (
|
|
359
|
+
BRAIN_CORE.conversations
|
|
360
|
+
if BRAIN_CORE is not None
|
|
361
|
+
else ConversationStore(DATA_DIR / "knowledge_graph.sqlite")
|
|
362
|
+
)
|
|
355
363
|
CONVERSATIONS.import_legacy_json(HISTORY_FILE)
|
|
356
364
|
# Hooks registry is constructed here (ahead of the watcher) so folder-watch
|
|
357
365
|
# reindexes can fire the pre_index/post_index lifecycle hooks.
|
|
@@ -1249,6 +1257,13 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
1249
1257
|
except Exception as e:
|
|
1250
1258
|
return {"error": str(e)}
|
|
1251
1259
|
|
|
1260
|
+
def _product_hardening_status():
|
|
1261
|
+
return build_product_hardening_status(
|
|
1262
|
+
config=CONFIG,
|
|
1263
|
+
portability=KG_PORTABILITY,
|
|
1264
|
+
device_identity=DEVICE_IDENTITY,
|
|
1265
|
+
)
|
|
1266
|
+
|
|
1252
1267
|
app.include_router(create_admin_router(
|
|
1253
1268
|
require_admin=require_admin, require_user=require_user,
|
|
1254
1269
|
load_users=load_users, save_users=save_users,
|
|
@@ -1263,6 +1278,7 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
1263
1278
|
invite_code=INVITE_CODE, invite_gate_enabled=INVITE_GATE_ENABLED,
|
|
1264
1279
|
default_port=DEFAULT_PORT,
|
|
1265
1280
|
policy_matrix=policy_matrix,
|
|
1281
|
+
product_hardening_status=_product_hardening_status,
|
|
1266
1282
|
))
|
|
1267
1283
|
|
|
1268
1284
|
app.include_router(create_invitations_router(
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
v4 home for the brain's storage modules. The knowledge-graph store itself
|
|
4
|
-
still lives in the root ``knowledge_graph`` module pending its decomposition
|
|
5
|
-
(T3d); new brain components land here first.
|
|
6
|
-
"""
|
|
1
|
+
"""Compatibility namespace for the standalone :mod:`lattice_brain` package."""
|
|
7
2
|
|
|
3
|
+
from lattice_brain.core import BrainCore, BrainCoreConfig
|
|
8
4
|
from latticeai.brain.context import AssembledContext, ContextAssembler, ContextSection
|
|
9
5
|
from latticeai.brain.conversations import ConversationStore
|
|
10
6
|
from latticeai.brain.memory import BrainMemory
|
|
7
|
+
from latticeai.brain.store import KnowledgeGraphStore
|
|
11
8
|
|
|
12
9
|
__all__ = [
|
|
13
10
|
"AssembledContext",
|
|
11
|
+
"BrainCore",
|
|
12
|
+
"BrainCoreConfig",
|
|
14
13
|
"BrainMemory",
|
|
15
14
|
"ContextAssembler",
|
|
16
15
|
"ContextSection",
|
|
17
16
|
"ConversationStore",
|
|
17
|
+
"KnowledgeGraphStore",
|
|
18
18
|
]
|
|
@@ -33,7 +33,7 @@ except Exception: # pragma: no cover - v2 schema is optional at import time
|
|
|
33
33
|
EdgeType = None # type: ignore[assignment]
|
|
34
34
|
_exec_script = None # type: ignore[assignment]
|
|
35
35
|
|
|
36
|
-
from
|
|
36
|
+
from lattice_brain.embeddings import LocalEmbeddingModel
|
|
37
37
|
|
|
38
38
|
# Default read source for the graph queries: v2 reconstruction views.
|
|
39
39
|
# Override with LATTICEAI_KG_READ_V2=0 to fall back to the legacy tables.
|
|
@@ -23,7 +23,7 @@ import uuid
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Any, Dict, List, Optional
|
|
25
25
|
|
|
26
|
-
from
|
|
26
|
+
from lattice_brain.identity import DeviceIdentity, fingerprint_of, verify_signature
|
|
27
27
|
|
|
28
28
|
PEER_AUTH_WINDOW_SECONDS = 300
|
|
29
29
|
_NONCE_CACHE_MAX = 4096
|
|
@@ -813,6 +813,15 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
813
813
|
raise
|
|
814
814
|
|
|
815
815
|
def index_status(self) -> Dict[str, Any]:
|
|
816
|
+
storage_capabilities = None
|
|
817
|
+
try:
|
|
818
|
+
storage_capabilities = self.storage_engine.capabilities().as_dict()
|
|
819
|
+
except Exception as exc:
|
|
820
|
+
storage_capabilities = {
|
|
821
|
+
"engine": "sqlite",
|
|
822
|
+
"available": False,
|
|
823
|
+
"reason": str(exc),
|
|
824
|
+
}
|
|
816
825
|
with self._connect() as conn:
|
|
817
826
|
vector_counts = {
|
|
818
827
|
row["item_type"]: row["count"]
|
|
@@ -864,6 +873,12 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
864
873
|
# Honest capability report: trigram FTS5 keyword index, or
|
|
865
874
|
# LIKE-scan fallback when this SQLite build lacks it.
|
|
866
875
|
"fts_enabled": bool(getattr(self, "_fts_enabled", False)),
|
|
876
|
+
"engine": storage_capabilities,
|
|
877
|
+
"vector_search_backend": (
|
|
878
|
+
storage_capabilities.get("vector_backend")
|
|
879
|
+
if isinstance(storage_capabilities, dict)
|
|
880
|
+
else "bruteforce-cosine"
|
|
881
|
+
),
|
|
867
882
|
},
|
|
868
883
|
"source_items": len(source_items),
|
|
869
884
|
"indexed_items": sum(vector_counts.values()),
|
package/latticeai/brain/store.py
CHANGED
|
@@ -21,11 +21,31 @@ class KnowledgeGraphStore(
|
|
|
21
21
|
KnowledgeGraphDocumentsMixin,
|
|
22
22
|
KnowledgeGraphRetrievalMixin,
|
|
23
23
|
):
|
|
24
|
-
def __init__(
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
db_path: Path,
|
|
27
|
+
blob_dir: Path,
|
|
28
|
+
embedder: Any = None,
|
|
29
|
+
storage_engine: Any = None,
|
|
30
|
+
):
|
|
25
31
|
self.db_path = Path(db_path)
|
|
26
32
|
self.blob_dir = Path(blob_dir)
|
|
27
33
|
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
28
34
|
self.blob_dir.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
if storage_engine is None:
|
|
36
|
+
from lattice_brain.storage import SQLiteEngine
|
|
37
|
+
|
|
38
|
+
storage_engine = SQLiteEngine(self.db_path)
|
|
39
|
+
storage_caps = storage_engine.capabilities()
|
|
40
|
+
if not storage_caps.available:
|
|
41
|
+
raise RuntimeError(storage_caps.reason or "Brain storage is unavailable.")
|
|
42
|
+
if storage_caps.engine != "sqlite":
|
|
43
|
+
raise RuntimeError(
|
|
44
|
+
"KnowledgeGraphStore currently requires SQLiteEngine. "
|
|
45
|
+
"Explicit non-SQLite storage must use the migration/scale tooling; "
|
|
46
|
+
"no SQLite fallback is attempted."
|
|
47
|
+
)
|
|
48
|
+
self.storage_engine = storage_engine
|
|
29
49
|
# The embedder is swappable behind a fixed interface
|
|
30
50
|
# (model_id/dim/embed/encode/decode/similarity). Defaults to the
|
|
31
51
|
# deterministic, offline hash model so the store works with no config;
|
|
@@ -49,11 +69,7 @@ class KnowledgeGraphStore(
|
|
|
49
69
|
return ("nodes", "edges")
|
|
50
70
|
|
|
51
71
|
def _connect(self) -> sqlite3.Connection:
|
|
52
|
-
|
|
53
|
-
conn.row_factory = sqlite3.Row
|
|
54
|
-
conn.execute("PRAGMA journal_mode=WAL")
|
|
55
|
-
conn.execute("PRAGMA foreign_keys=ON")
|
|
56
|
-
return conn
|
|
72
|
+
return self.storage_engine.connect()
|
|
57
73
|
|
|
58
74
|
def _init_db(self) -> None:
|
|
59
75
|
with self._connect() as conn:
|
package/latticeai/core/config.py
CHANGED
|
@@ -104,6 +104,11 @@ class Config:
|
|
|
104
104
|
embedding_timeout: int
|
|
105
105
|
embedding_custom_target: str
|
|
106
106
|
|
|
107
|
+
# ── brain storage ───────────────────────────────────────────────
|
|
108
|
+
storage_engine: str
|
|
109
|
+
postgres_dsn: str
|
|
110
|
+
postgres_schema: str
|
|
111
|
+
|
|
107
112
|
# ── SSO / OIDC ──────────────────────────────────────────────────
|
|
108
113
|
sso_discovery_url: str
|
|
109
114
|
sso_client_id: str
|
|
@@ -158,7 +163,7 @@ class Config:
|
|
|
158
163
|
host=host,
|
|
159
164
|
port=port,
|
|
160
165
|
network_exposed=network_exposed,
|
|
161
|
-
enable_telegram=_bool(env, "LATTICEAI_ENABLE_TELEGRAM", default=
|
|
166
|
+
enable_telegram=_bool(env, "LATTICEAI_ENABLE_TELEGRAM", default=False),
|
|
162
167
|
enable_graph=_bool(env, "LATTICEAI_ENABLE_GRAPH", default=True),
|
|
163
168
|
autoload_models=_bool(env, "LATTICEAI_AUTOLOAD_MODELS", default=is_public),
|
|
164
169
|
model_idle_unload_seconds=_int(env, "LATTICEAI_MODEL_IDLE_UNLOAD_SECONDS", 0),
|
|
@@ -185,6 +190,9 @@ class Config:
|
|
|
185
190
|
embedding_dim=_int(env, "LATTICEAI_VECTOR_DIM", 0),
|
|
186
191
|
embedding_timeout=_int(env, "LATTICEAI_EMBEDDING_TIMEOUT", 30),
|
|
187
192
|
embedding_custom_target=_value(env, "LATTICEAI_EMBEDDING_CUSTOM_TARGET", ""),
|
|
193
|
+
storage_engine=_value(env, "LATTICEAI_STORAGE_ENGINE", "sqlite").strip().lower() or "sqlite",
|
|
194
|
+
postgres_dsn=_value(env, "LATTICEAI_POSTGRES_DSN", ""),
|
|
195
|
+
postgres_schema=_value(env, "LATTICEAI_POSTGRES_SCHEMA", "lattice_brain"),
|
|
188
196
|
sso_discovery_url=_value(env, "OIDC_DISCOVERY_URL", ""),
|
|
189
197
|
sso_client_id=_value(env, "OIDC_CLIENT_ID", ""),
|
|
190
198
|
sso_client_secret=_value(env, "OIDC_CLIENT_SECRET", ""),
|