ltcai 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -24
- package/docs/CHANGELOG.md +42 -0
- package/docs/V4_2_BRAIN_CORE_ARCHITECTURE.md +97 -0
- package/docs/V4_2_STORAGE_MIGRATION_REPORT.md +91 -0
- package/docs/V4_2_VALIDATION_REPORT.md +89 -0
- package/docs/V4_DIGITAL_BRAIN_RECOVERY.md +31 -33
- package/frontend/openapi.json +247 -1
- package/frontend/src/api/client.ts +3 -0
- package/frontend/src/api/openapi.ts +284 -0
- package/frontend/src/pages/System.tsx +34 -0
- package/kg_schema.py +1 -1
- package/knowledge_graph.py +4 -4
- package/lattice_brain/__init__.py +70 -0
- package/lattice_brain/_kg_common.py +1 -0
- package/lattice_brain/archive.py +133 -0
- package/lattice_brain/context.py +3 -0
- package/lattice_brain/conversations.py +3 -0
- package/lattice_brain/core.py +82 -0
- package/lattice_brain/discovery.py +1 -0
- package/lattice_brain/documents.py +1 -0
- package/lattice_brain/embeddings.py +82 -0
- package/lattice_brain/identity.py +13 -0
- package/lattice_brain/ingest.py +1 -0
- package/lattice_brain/memory.py +3 -0
- package/lattice_brain/network.py +1 -0
- package/lattice_brain/projection.py +1 -0
- package/lattice_brain/provenance.py +1 -0
- package/lattice_brain/retrieval.py +1 -0
- package/lattice_brain/schema.py +1 -0
- package/lattice_brain/storage/__init__.py +22 -0
- package/lattice_brain/storage/base.py +72 -0
- package/lattice_brain/storage/docker.py +105 -0
- package/lattice_brain/storage/factory.py +31 -0
- package/lattice_brain/storage/migration.py +190 -0
- package/lattice_brain/storage/postgres.py +123 -0
- package/lattice_brain/storage/sqlite.py +128 -0
- package/lattice_brain/store.py +3 -0
- package/lattice_brain/write_master.py +1 -0
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/portability.py +69 -0
- package/latticeai/app_factory.py +17 -10
- package/latticeai/brain/__init__.py +6 -6
- package/latticeai/brain/_kg_common.py +1 -1
- package/latticeai/brain/network.py +1 -1
- package/latticeai/brain/retrieval.py +15 -0
- package/latticeai/brain/store.py +22 -6
- package/latticeai/core/config.py +8 -0
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/multi_agent.py +1 -1
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/services/kg_portability.py +82 -1
- package/package.json +2 -1
- package/scripts/bump_version.py +3 -0
- package/scripts/lint_frontend.mjs +5 -0
- package/scripts/migrate_brain_storage.py +53 -0
- package/scripts/wheel_smoke.py +3 -0
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +5 -2
- package/static/app/asset-manifest.json +5 -5
- package/static/app/assets/index-CDjiH_se.css +2 -0
- package/static/app/assets/{index-CJRAzNnf.js → index-C_HAkbAg.js} +3 -3
- package/static/app/assets/index-C_HAkbAg.js.map +1 -0
- package/static/app/index.html +2 -2
- package/static/app/assets/index-CJRAzNnf.js.map +0 -1
- package/static/app/assets/index-CSwBBgf4.css +0 -2
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Safe SQLite to Postgres migration tooling."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sqlite3
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, List
|
|
9
|
+
|
|
10
|
+
from .postgres import PostgresEngine, _quote_ident
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _pg_type(sqlite_type: str) -> str:
|
|
14
|
+
t = str(sqlite_type or "").upper()
|
|
15
|
+
if "INT" in t:
|
|
16
|
+
return "bigint"
|
|
17
|
+
if any(token in t for token in ("REAL", "FLOA", "DOUB")):
|
|
18
|
+
return "double precision"
|
|
19
|
+
if "BLOB" in t:
|
|
20
|
+
return "bytea"
|
|
21
|
+
return "text"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _adapt_value(value: Any) -> Any:
|
|
25
|
+
if isinstance(value, memoryview):
|
|
26
|
+
return bytes(value)
|
|
27
|
+
return value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class TablePlan:
|
|
32
|
+
name: str
|
|
33
|
+
columns: List[Dict[str, str]]
|
|
34
|
+
rows: int
|
|
35
|
+
conflict_key: str
|
|
36
|
+
conflict_columns: List[str]
|
|
37
|
+
rowid_available: bool
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class SQLiteToPostgresMigrator:
|
|
41
|
+
"""Copies every user table from a Lattice SQLite brain into Postgres.
|
|
42
|
+
|
|
43
|
+
The migration is idempotent: tables with an ``id`` column upsert on ``id``;
|
|
44
|
+
tables without one use the preserved SQLite rowid in ``__source_rowid``.
|
|
45
|
+
SQLite remains untouched throughout.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(self, sqlite_path: Path, target: PostgresEngine) -> None:
|
|
49
|
+
self.sqlite_path = Path(sqlite_path)
|
|
50
|
+
self.target = target
|
|
51
|
+
|
|
52
|
+
def plan(self) -> Dict[str, Any]:
|
|
53
|
+
if not self.sqlite_path.exists():
|
|
54
|
+
raise FileNotFoundError(f"SQLite brain database not found: {self.sqlite_path}")
|
|
55
|
+
with sqlite3.connect(str(self.sqlite_path)) as conn:
|
|
56
|
+
conn.row_factory = sqlite3.Row
|
|
57
|
+
table_names = [
|
|
58
|
+
row["name"]
|
|
59
|
+
for row in conn.execute(
|
|
60
|
+
"""
|
|
61
|
+
SELECT name FROM sqlite_master
|
|
62
|
+
WHERE type='table' AND name NOT LIKE 'sqlite_%'
|
|
63
|
+
ORDER BY name
|
|
64
|
+
"""
|
|
65
|
+
)
|
|
66
|
+
]
|
|
67
|
+
tables = []
|
|
68
|
+
for table in table_names:
|
|
69
|
+
cols = [
|
|
70
|
+
{"name": row["name"], "type": row["type"] or "TEXT"}
|
|
71
|
+
for row in conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})")
|
|
72
|
+
]
|
|
73
|
+
row_count = conn.execute(f"SELECT COUNT(*) FROM {_quote_sqlite_ident(table)}").fetchone()[0]
|
|
74
|
+
names = {c["name"] for c in cols}
|
|
75
|
+
rowid_available = _rowid_available(conn, table)
|
|
76
|
+
pk_columns = [
|
|
77
|
+
row["name"]
|
|
78
|
+
for row in sorted(
|
|
79
|
+
conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})"),
|
|
80
|
+
key=lambda item: int(item["pk"] or 0),
|
|
81
|
+
)
|
|
82
|
+
if int(row["pk"] or 0) > 0
|
|
83
|
+
]
|
|
84
|
+
conflict_columns = (
|
|
85
|
+
["id"]
|
|
86
|
+
if "id" in names
|
|
87
|
+
else pk_columns
|
|
88
|
+
if pk_columns
|
|
89
|
+
else ["__source_rowid"]
|
|
90
|
+
if rowid_available
|
|
91
|
+
else []
|
|
92
|
+
)
|
|
93
|
+
if not conflict_columns:
|
|
94
|
+
raise RuntimeError(
|
|
95
|
+
f"Cannot safely migrate rowid-less SQLite table without a primary key: {table}"
|
|
96
|
+
)
|
|
97
|
+
tables.append(
|
|
98
|
+
TablePlan(
|
|
99
|
+
name=table,
|
|
100
|
+
columns=cols,
|
|
101
|
+
rows=int(row_count),
|
|
102
|
+
conflict_key=conflict_columns[0],
|
|
103
|
+
conflict_columns=conflict_columns,
|
|
104
|
+
rowid_available=rowid_available,
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
return {
|
|
108
|
+
"source": str(self.sqlite_path),
|
|
109
|
+
"target_engine": self.target.name,
|
|
110
|
+
"target_schema": self.target.config.schema,
|
|
111
|
+
"tables": [table.__dict__ for table in tables],
|
|
112
|
+
"total_rows": sum(table.rows for table in tables),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
def migrate(self, *, dry_run: bool = False) -> Dict[str, Any]:
|
|
116
|
+
plan = self.plan()
|
|
117
|
+
if dry_run:
|
|
118
|
+
return {"status": "planned", **plan}
|
|
119
|
+
schema = _quote_ident(self.target.config.schema)
|
|
120
|
+
copied: Dict[str, int] = {}
|
|
121
|
+
self.target.initialize()
|
|
122
|
+
with sqlite3.connect(str(self.sqlite_path)) as src, self.target.connect() as dst:
|
|
123
|
+
src.row_factory = sqlite3.Row
|
|
124
|
+
with dst.cursor() as cur:
|
|
125
|
+
for table in plan["tables"]:
|
|
126
|
+
name = str(table["name"])
|
|
127
|
+
cols = list(table["columns"])
|
|
128
|
+
conflict_columns = list(table.get("conflict_columns") or [table["conflict_key"]])
|
|
129
|
+
rowid_available = bool(table.get("rowid_available", True))
|
|
130
|
+
pg_table = f"{schema}.{_quote_ident(name)}"
|
|
131
|
+
defs = ["__source_rowid bigint NOT NULL"] if rowid_available else []
|
|
132
|
+
for col in cols:
|
|
133
|
+
defs.append(f"{_quote_ident(col['name'])} {_pg_type(col['type'])}")
|
|
134
|
+
pk = ", ".join(_quote_ident(c) for c in conflict_columns)
|
|
135
|
+
cur.execute(
|
|
136
|
+
f"CREATE TABLE IF NOT EXISTS {pg_table} ({', '.join(defs)}, PRIMARY KEY ({pk}))"
|
|
137
|
+
)
|
|
138
|
+
if rowid_available:
|
|
139
|
+
select_sql = (
|
|
140
|
+
f"SELECT rowid AS __source_rowid, * FROM {_quote_sqlite_ident(name)} ORDER BY rowid"
|
|
141
|
+
)
|
|
142
|
+
else:
|
|
143
|
+
order_by = ", ".join(_quote_sqlite_ident(c) for c in conflict_columns)
|
|
144
|
+
select_sql = f"SELECT * FROM {_quote_sqlite_ident(name)} ORDER BY {order_by}"
|
|
145
|
+
rows = src.execute(select_sql).fetchall()
|
|
146
|
+
if not rows:
|
|
147
|
+
copied[name] = 0
|
|
148
|
+
continue
|
|
149
|
+
columns = (["__source_rowid"] if rowid_available else []) + [c["name"] for c in cols]
|
|
150
|
+
placeholders = ", ".join(["%s"] * len(columns))
|
|
151
|
+
quoted_columns = ", ".join(_quote_ident(c) for c in columns)
|
|
152
|
+
updates = ", ".join(
|
|
153
|
+
f"{_quote_ident(c)} = EXCLUDED.{_quote_ident(c)}"
|
|
154
|
+
for c in columns
|
|
155
|
+
if c not in conflict_columns
|
|
156
|
+
)
|
|
157
|
+
conflict_action = f"DO UPDATE SET {updates}" if updates else "DO NOTHING"
|
|
158
|
+
sql = (
|
|
159
|
+
f"INSERT INTO {pg_table} ({quoted_columns}) VALUES ({placeholders}) "
|
|
160
|
+
f"ON CONFLICT ({pk}) {conflict_action}"
|
|
161
|
+
)
|
|
162
|
+
cur.executemany(
|
|
163
|
+
sql,
|
|
164
|
+
[
|
|
165
|
+
tuple(_adapt_value(row[col]) for col in columns)
|
|
166
|
+
for row in rows
|
|
167
|
+
],
|
|
168
|
+
)
|
|
169
|
+
copied[name] = len(rows)
|
|
170
|
+
return {
|
|
171
|
+
"status": "migrated",
|
|
172
|
+
**plan,
|
|
173
|
+
"copied_rows": copied,
|
|
174
|
+
"total_copied_rows": sum(copied.values()),
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _quote_sqlite_ident(value: str) -> str:
|
|
179
|
+
return '"' + str(value).replace('"', '""') + '"'
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _rowid_available(conn: sqlite3.Connection, table: str) -> bool:
|
|
183
|
+
try:
|
|
184
|
+
conn.execute(f"SELECT rowid FROM {_quote_sqlite_ident(table)} LIMIT 1").fetchall()
|
|
185
|
+
return True
|
|
186
|
+
except sqlite3.OperationalError:
|
|
187
|
+
return False
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
__all__ = ["SQLiteToPostgresMigrator", "TablePlan"]
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Opt-in Postgres/pgvector storage engine for Brain Core scale mode."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict
|
|
8
|
+
|
|
9
|
+
from .base import StorageCapabilities, StorageEngine, StorageUnavailable
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _quote_ident(value: str) -> str:
|
|
13
|
+
return '"' + str(value).replace('"', '""') + '"'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class PostgresConfig:
|
|
18
|
+
dsn: str
|
|
19
|
+
schema: str = "lattice_brain"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PostgresEngine(StorageEngine):
|
|
23
|
+
name = "postgres"
|
|
24
|
+
|
|
25
|
+
def __init__(self, dsn: str, *, schema: str = "lattice_brain") -> None:
|
|
26
|
+
self.config = PostgresConfig(dsn=dsn, schema=schema)
|
|
27
|
+
|
|
28
|
+
def _psycopg(self):
|
|
29
|
+
try:
|
|
30
|
+
import psycopg # type: ignore
|
|
31
|
+
except Exception as exc:
|
|
32
|
+
raise StorageUnavailable(
|
|
33
|
+
"Postgres storage requires optional dependency 'psycopg'. "
|
|
34
|
+
"Install the postgres extra before selecting LATTICEAI_STORAGE_ENGINE=postgres."
|
|
35
|
+
) from exc
|
|
36
|
+
return psycopg
|
|
37
|
+
|
|
38
|
+
def connect(self) -> Any:
|
|
39
|
+
if not self.config.dsn:
|
|
40
|
+
raise StorageUnavailable(
|
|
41
|
+
"Postgres storage requires LATTICEAI_POSTGRES_DSN; no SQLite fallback is attempted."
|
|
42
|
+
)
|
|
43
|
+
psycopg = self._psycopg()
|
|
44
|
+
return psycopg.connect(self.config.dsn)
|
|
45
|
+
|
|
46
|
+
def initialize(self) -> Dict[str, Any]:
|
|
47
|
+
schema = _quote_ident(self.config.schema)
|
|
48
|
+
with self.connect() as conn:
|
|
49
|
+
with conn.cursor() as cur:
|
|
50
|
+
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
51
|
+
cur.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}")
|
|
52
|
+
cur.execute(
|
|
53
|
+
f"""
|
|
54
|
+
CREATE TABLE IF NOT EXISTS {schema}.storage_meta (
|
|
55
|
+
key text PRIMARY KEY,
|
|
56
|
+
value text NOT NULL,
|
|
57
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
58
|
+
)
|
|
59
|
+
"""
|
|
60
|
+
)
|
|
61
|
+
cur.execute(
|
|
62
|
+
f"""
|
|
63
|
+
CREATE TABLE IF NOT EXISTS {schema}.brain_vectors (
|
|
64
|
+
item_id text PRIMARY KEY,
|
|
65
|
+
item_type text NOT NULL,
|
|
66
|
+
source_node text NOT NULL,
|
|
67
|
+
text_hash text NOT NULL,
|
|
68
|
+
embedding vector,
|
|
69
|
+
embedding_dim integer NOT NULL,
|
|
70
|
+
embedding_model text NOT NULL,
|
|
71
|
+
metadata_json jsonb NOT NULL DEFAULT '{{}}'::jsonb,
|
|
72
|
+
indexed_at timestamptz NOT NULL DEFAULT now()
|
|
73
|
+
)
|
|
74
|
+
"""
|
|
75
|
+
)
|
|
76
|
+
cur.execute(
|
|
77
|
+
f"""
|
|
78
|
+
INSERT INTO {schema}.storage_meta(key, value)
|
|
79
|
+
VALUES ('engine', 'postgres')
|
|
80
|
+
ON CONFLICT (key) DO UPDATE
|
|
81
|
+
SET value = EXCLUDED.value, updated_at = now()
|
|
82
|
+
"""
|
|
83
|
+
)
|
|
84
|
+
return {"engine": self.name, "schema": self.config.schema}
|
|
85
|
+
|
|
86
|
+
def capabilities(self) -> StorageCapabilities:
|
|
87
|
+
try:
|
|
88
|
+
with self.connect() as conn:
|
|
89
|
+
with conn.cursor() as cur:
|
|
90
|
+
cur.execute("SELECT extname FROM pg_extension WHERE extname='vector'")
|
|
91
|
+
pgvector = cur.fetchone() is not None
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
return StorageCapabilities(
|
|
94
|
+
engine=self.name,
|
|
95
|
+
available=False,
|
|
96
|
+
reason=str(exc),
|
|
97
|
+
vector_backend="pgvector",
|
|
98
|
+
metadata={"schema": self.config.schema},
|
|
99
|
+
)
|
|
100
|
+
return StorageCapabilities(
|
|
101
|
+
engine=self.name,
|
|
102
|
+
available=True,
|
|
103
|
+
reason=None if pgvector else "pgvector extension is not installed",
|
|
104
|
+
vector_backend="pgvector",
|
|
105
|
+
vector_available=pgvector,
|
|
106
|
+
backup_restore=False,
|
|
107
|
+
migrations=True,
|
|
108
|
+
encrypted_archives=False,
|
|
109
|
+
metadata={"schema": self.config.schema},
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
def backup(self, destination: Path) -> Dict[str, Any]:
|
|
113
|
+
raise StorageUnavailable(
|
|
114
|
+
"Postgres logical backup is not implemented inside the app; use pg_dump for this engine."
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def restore(self, source: Path) -> Dict[str, Any]:
|
|
118
|
+
raise StorageUnavailable(
|
|
119
|
+
"Postgres restore is not implemented inside the app; use pg_restore/psql for this engine."
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
__all__ = ["PostgresConfig", "PostgresEngine", "_quote_ident"]
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""SQLite storage engine for Brain Core.
|
|
2
|
+
|
|
3
|
+
SQLite is the default and remains fully local-first. sqlite-vec is detected and
|
|
4
|
+
loaded when present; otherwise the existing brute-force cosine path remains the
|
|
5
|
+
honest, real fallback and is surfaced in capability reports.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import shutil
|
|
11
|
+
import sqlite3
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Dict, Optional
|
|
14
|
+
|
|
15
|
+
from .base import StorageCapabilities, StorageEngine
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _load_sqlite_vec(conn: sqlite3.Connection) -> tuple[bool, Optional[str]]:
|
|
19
|
+
try:
|
|
20
|
+
import sqlite_vec # type: ignore
|
|
21
|
+
except Exception as exc:
|
|
22
|
+
return False, f"sqlite-vec Python package not installed: {exc}"
|
|
23
|
+
try:
|
|
24
|
+
conn.enable_load_extension(True)
|
|
25
|
+
except Exception:
|
|
26
|
+
pass
|
|
27
|
+
try:
|
|
28
|
+
sqlite_vec.load(conn)
|
|
29
|
+
except Exception as exc:
|
|
30
|
+
return False, f"sqlite-vec extension failed to load: {exc}"
|
|
31
|
+
return True, None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class SQLiteEngine(StorageEngine):
|
|
35
|
+
name = "sqlite"
|
|
36
|
+
|
|
37
|
+
def __init__(self, db_path: Path, *, load_vec: bool = True) -> None:
|
|
38
|
+
self.db_path = Path(db_path)
|
|
39
|
+
self.load_vec = bool(load_vec)
|
|
40
|
+
self._sqlite_vec_loaded = False
|
|
41
|
+
self._sqlite_vec_reason: Optional[str] = None
|
|
42
|
+
|
|
43
|
+
def connect(self) -> sqlite3.Connection:
|
|
44
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
conn = sqlite3.connect(str(self.db_path))
|
|
46
|
+
conn.row_factory = sqlite3.Row
|
|
47
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
48
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
49
|
+
if self.load_vec:
|
|
50
|
+
loaded, reason = _load_sqlite_vec(conn)
|
|
51
|
+
self._sqlite_vec_loaded = loaded
|
|
52
|
+
self._sqlite_vec_reason = reason
|
|
53
|
+
return conn
|
|
54
|
+
|
|
55
|
+
def initialize(self) -> Dict[str, Any]:
|
|
56
|
+
with self.connect() as conn:
|
|
57
|
+
conn.execute(
|
|
58
|
+
"""
|
|
59
|
+
CREATE TABLE IF NOT EXISTS storage_meta (
|
|
60
|
+
key TEXT PRIMARY KEY,
|
|
61
|
+
value TEXT NOT NULL
|
|
62
|
+
)
|
|
63
|
+
"""
|
|
64
|
+
)
|
|
65
|
+
conn.execute(
|
|
66
|
+
"INSERT OR REPLACE INTO storage_meta(key, value) VALUES ('engine', 'sqlite')"
|
|
67
|
+
)
|
|
68
|
+
return {"engine": self.name, "db_path": str(self.db_path)}
|
|
69
|
+
|
|
70
|
+
def capabilities(self) -> StorageCapabilities:
|
|
71
|
+
if not self.db_path.parent.exists():
|
|
72
|
+
return StorageCapabilities(
|
|
73
|
+
engine=self.name,
|
|
74
|
+
available=True,
|
|
75
|
+
vector_backend="bruteforce-cosine",
|
|
76
|
+
vector_available=True,
|
|
77
|
+
backup_restore=True,
|
|
78
|
+
migrations=True,
|
|
79
|
+
encrypted_archives=True,
|
|
80
|
+
metadata={"db_path": str(self.db_path), "sqlite_vec_loaded": False},
|
|
81
|
+
)
|
|
82
|
+
# Probe on demand so status is accurate even before the graph opens.
|
|
83
|
+
try:
|
|
84
|
+
with self.connect():
|
|
85
|
+
pass
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
return StorageCapabilities(
|
|
88
|
+
engine=self.name,
|
|
89
|
+
available=False,
|
|
90
|
+
reason=str(exc),
|
|
91
|
+
metadata={"db_path": str(self.db_path)},
|
|
92
|
+
)
|
|
93
|
+
vector_backend = "sqlite-vec" if self._sqlite_vec_loaded else "bruteforce-cosine"
|
|
94
|
+
return StorageCapabilities(
|
|
95
|
+
engine=self.name,
|
|
96
|
+
available=True,
|
|
97
|
+
reason=None if self._sqlite_vec_loaded else self._sqlite_vec_reason,
|
|
98
|
+
vector_backend=vector_backend,
|
|
99
|
+
vector_available=True,
|
|
100
|
+
backup_restore=True,
|
|
101
|
+
migrations=True,
|
|
102
|
+
encrypted_archives=True,
|
|
103
|
+
metadata={
|
|
104
|
+
"db_path": str(self.db_path),
|
|
105
|
+
"sqlite_vec_loaded": self._sqlite_vec_loaded,
|
|
106
|
+
},
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def backup(self, destination: Path) -> Dict[str, Any]:
|
|
110
|
+
dest = Path(destination)
|
|
111
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
with self.connect() as src, sqlite3.connect(str(dest)) as dst:
|
|
113
|
+
src.backup(dst)
|
|
114
|
+
return {"engine": self.name, "path": str(dest), "bytes": dest.stat().st_size}
|
|
115
|
+
|
|
116
|
+
def restore(self, source: Path) -> Dict[str, Any]:
|
|
117
|
+
src = Path(source)
|
|
118
|
+
if not src.exists():
|
|
119
|
+
raise FileNotFoundError(f"SQLite backup not found: {src}")
|
|
120
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
121
|
+
for sibling in (self.db_path, Path(str(self.db_path) + "-wal"), Path(str(self.db_path) + "-shm")):
|
|
122
|
+
if sibling.exists():
|
|
123
|
+
sibling.unlink()
|
|
124
|
+
shutil.copyfile(src, self.db_path)
|
|
125
|
+
return {"engine": self.name, "restored": True, "path": str(self.db_path)}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
__all__ = ["SQLiteEngine"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from latticeai.brain.write_master import * # noqa: F401,F403
|
package/latticeai/__init__.py
CHANGED
|
@@ -28,6 +28,28 @@ class RestoreRequest(BaseModel):
|
|
|
28
28
|
verify: bool = True
|
|
29
29
|
|
|
30
30
|
|
|
31
|
+
class EncryptedArchiveRequest(BaseModel):
|
|
32
|
+
path: Optional[str] = None
|
|
33
|
+
passphrase: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EncryptedRestoreRequest(BaseModel):
|
|
37
|
+
path: str
|
|
38
|
+
passphrase: str
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class DockerPostgresRequest(BaseModel):
|
|
42
|
+
consent: bool = False
|
|
43
|
+
dry_run: bool = False
|
|
44
|
+
port: int = 5432
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class SQLiteToPostgresRequest(BaseModel):
|
|
48
|
+
dsn: str
|
|
49
|
+
schema_name: str = "lattice_brain"
|
|
50
|
+
dry_run: bool = True
|
|
51
|
+
|
|
52
|
+
|
|
31
53
|
def create_portability_router(
|
|
32
54
|
*,
|
|
33
55
|
service: Any,
|
|
@@ -46,6 +68,12 @@ def create_portability_router(
|
|
|
46
68
|
_require_service()
|
|
47
69
|
return service.snapshot_metadata()
|
|
48
70
|
|
|
71
|
+
@router.get("/api/brain/storage")
|
|
72
|
+
async def brain_storage_status(request: Request):
|
|
73
|
+
require_user(request)
|
|
74
|
+
_require_service()
|
|
75
|
+
return service.storage_status()
|
|
76
|
+
|
|
49
77
|
@router.get("/api/knowledge-graph/provenance")
|
|
50
78
|
async def recent_provenance(request: Request, limit: int = 50, source_type: Optional[str] = None):
|
|
51
79
|
"""Recent ingestions (provenance trail) for the ingestion-sources UI."""
|
|
@@ -90,4 +118,45 @@ def create_portability_router(
|
|
|
90
118
|
except (ValueError, FileNotFoundError) as exc:
|
|
91
119
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
92
120
|
|
|
121
|
+
@router.post("/api/knowledge-graph/archive")
|
|
122
|
+
async def encrypted_archive(req: EncryptedArchiveRequest, request: Request):
|
|
123
|
+
require_admin(request)
|
|
124
|
+
_require_service()
|
|
125
|
+
try:
|
|
126
|
+
return service.encrypted_archive(req.path, passphrase=req.passphrase)
|
|
127
|
+
except (ValueError, FileNotFoundError) as exc:
|
|
128
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
129
|
+
|
|
130
|
+
@router.post("/api/knowledge-graph/archive/restore")
|
|
131
|
+
async def restore_encrypted_archive(req: EncryptedRestoreRequest, request: Request):
|
|
132
|
+
require_admin(request)
|
|
133
|
+
_require_service()
|
|
134
|
+
try:
|
|
135
|
+
return service.restore_encrypted_archive(req.path, passphrase=req.passphrase)
|
|
136
|
+
except (ValueError, FileNotFoundError) as exc:
|
|
137
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
138
|
+
|
|
139
|
+
@router.post("/api/brain/storage/postgres/docker")
|
|
140
|
+
async def setup_postgres_docker(req: DockerPostgresRequest, request: Request):
|
|
141
|
+
require_admin(request)
|
|
142
|
+
_require_service()
|
|
143
|
+
return service.postgres_docker_setup(
|
|
144
|
+
consent=req.consent,
|
|
145
|
+
dry_run=req.dry_run,
|
|
146
|
+
port=req.port,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
@router.post("/api/brain/storage/migrate-postgres")
|
|
150
|
+
async def migrate_sqlite_to_postgres(req: SQLiteToPostgresRequest, request: Request):
|
|
151
|
+
require_admin(request)
|
|
152
|
+
_require_service()
|
|
153
|
+
try:
|
|
154
|
+
return service.migrate_sqlite_to_postgres(
|
|
155
|
+
dsn=req.dsn,
|
|
156
|
+
schema=req.schema_name,
|
|
157
|
+
dry_run=req.dry_run,
|
|
158
|
+
)
|
|
159
|
+
except (ValueError, FileNotFoundError, RuntimeError) as exc:
|
|
160
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
161
|
+
|
|
93
162
|
return router
|
package/latticeai/app_factory.py
CHANGED
|
@@ -60,7 +60,7 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
60
60
|
from pydantic import BaseModel
|
|
61
61
|
|
|
62
62
|
from latticeai.models.router import LLMRouter, normalize_branding
|
|
63
|
-
from knowledge_graph import
|
|
63
|
+
from knowledge_graph import set_llm_router
|
|
64
64
|
from local_knowledge_api import LocalKnowledgeWatcher
|
|
65
65
|
from latticeai.core.security import (
|
|
66
66
|
hash_password,
|
|
@@ -161,11 +161,12 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
161
161
|
from latticeai.api.portability import create_portability_router
|
|
162
162
|
from latticeai.services.memory_service import MemoryService
|
|
163
163
|
from latticeai.services.ingestion import IngestionItem, IngestionPipeline
|
|
164
|
-
from
|
|
165
|
-
from
|
|
166
|
-
from
|
|
167
|
-
from
|
|
168
|
-
from
|
|
164
|
+
from lattice_brain import BrainCore, ConversationStore
|
|
165
|
+
from lattice_brain.storage import storage_from_env
|
|
166
|
+
from lattice_brain.context import ContextAssembler
|
|
167
|
+
from lattice_brain.memory import BrainMemory
|
|
168
|
+
from lattice_brain.identity import DeviceIdentity
|
|
169
|
+
from lattice_brain.network import BrainNetwork
|
|
169
170
|
from latticeai.api.network import create_network_router
|
|
170
171
|
from latticeai.services.kg_portability import KGPortabilityService
|
|
171
172
|
# The aliased names below look unused but are part of the legacy
|
|
@@ -342,16 +343,22 @@ def _build(config: "Optional[Config]" = None) -> Dict[str, Any]:
|
|
|
342
343
|
)
|
|
343
344
|
if EMBEDDER.fell_back:
|
|
344
345
|
logging.warning("Embedding provider %s unavailable: %s", EMBEDDER.requested, EMBEDDER.detail)
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
DATA_DIR
|
|
346
|
+
STORAGE_ENGINE = storage_from_env(os.environ, data_dir=DATA_DIR) if ENABLE_GRAPH else None
|
|
347
|
+
BRAIN_CORE = BrainCore.from_paths(
|
|
348
|
+
DATA_DIR,
|
|
348
349
|
embedder=EMBEDDER.provider,
|
|
350
|
+
storage_engine=STORAGE_ENGINE,
|
|
349
351
|
) if ENABLE_GRAPH else None
|
|
352
|
+
KNOWLEDGE_GRAPH = BRAIN_CORE.knowledge if BRAIN_CORE is not None else None
|
|
350
353
|
# ── v4 durable conversation store: unbounded episodic memory in the same
|
|
351
354
|
# SQLite file as the graph (kg_portability backup/restore covers it for
|
|
352
355
|
# free). Legacy chat_history.json is imported once, idempotently, and the
|
|
353
356
|
# file is left untouched on disk as the import source.
|
|
354
|
-
CONVERSATIONS =
|
|
357
|
+
CONVERSATIONS = (
|
|
358
|
+
BRAIN_CORE.conversations
|
|
359
|
+
if BRAIN_CORE is not None
|
|
360
|
+
else ConversationStore(DATA_DIR / "knowledge_graph.sqlite")
|
|
361
|
+
)
|
|
355
362
|
CONVERSATIONS.import_legacy_json(HISTORY_FILE)
|
|
356
363
|
# Hooks registry is constructed here (ahead of the watcher) so folder-watch
|
|
357
364
|
# reindexes can fire the pre_index/post_index lifecycle hooks.
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
v4 home for the brain's storage modules. The knowledge-graph store itself
|
|
4
|
-
still lives in the root ``knowledge_graph`` module pending its decomposition
|
|
5
|
-
(T3d); new brain components land here first.
|
|
6
|
-
"""
|
|
1
|
+
"""Compatibility namespace for the standalone :mod:`lattice_brain` package."""
|
|
7
2
|
|
|
3
|
+
from lattice_brain.core import BrainCore, BrainCoreConfig
|
|
8
4
|
from latticeai.brain.context import AssembledContext, ContextAssembler, ContextSection
|
|
9
5
|
from latticeai.brain.conversations import ConversationStore
|
|
10
6
|
from latticeai.brain.memory import BrainMemory
|
|
7
|
+
from latticeai.brain.store import KnowledgeGraphStore
|
|
11
8
|
|
|
12
9
|
__all__ = [
|
|
13
10
|
"AssembledContext",
|
|
11
|
+
"BrainCore",
|
|
12
|
+
"BrainCoreConfig",
|
|
14
13
|
"BrainMemory",
|
|
15
14
|
"ContextAssembler",
|
|
16
15
|
"ContextSection",
|
|
17
16
|
"ConversationStore",
|
|
17
|
+
"KnowledgeGraphStore",
|
|
18
18
|
]
|
|
@@ -33,7 +33,7 @@ except Exception: # pragma: no cover - v2 schema is optional at import time
|
|
|
33
33
|
EdgeType = None # type: ignore[assignment]
|
|
34
34
|
_exec_script = None # type: ignore[assignment]
|
|
35
35
|
|
|
36
|
-
from
|
|
36
|
+
from lattice_brain.embeddings import LocalEmbeddingModel
|
|
37
37
|
|
|
38
38
|
# Default read source for the graph queries: v2 reconstruction views.
|
|
39
39
|
# Override with LATTICEAI_KG_READ_V2=0 to fall back to the legacy tables.
|
|
@@ -23,7 +23,7 @@ import uuid
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Any, Dict, List, Optional
|
|
25
25
|
|
|
26
|
-
from
|
|
26
|
+
from lattice_brain.identity import DeviceIdentity, fingerprint_of, verify_signature
|
|
27
27
|
|
|
28
28
|
PEER_AUTH_WINDOW_SECONDS = 300
|
|
29
29
|
_NONCE_CACHE_MAX = 4096
|
|
@@ -813,6 +813,15 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
813
813
|
raise
|
|
814
814
|
|
|
815
815
|
def index_status(self) -> Dict[str, Any]:
|
|
816
|
+
storage_capabilities = None
|
|
817
|
+
try:
|
|
818
|
+
storage_capabilities = self.storage_engine.capabilities().as_dict()
|
|
819
|
+
except Exception as exc:
|
|
820
|
+
storage_capabilities = {
|
|
821
|
+
"engine": "sqlite",
|
|
822
|
+
"available": False,
|
|
823
|
+
"reason": str(exc),
|
|
824
|
+
}
|
|
816
825
|
with self._connect() as conn:
|
|
817
826
|
vector_counts = {
|
|
818
827
|
row["item_type"]: row["count"]
|
|
@@ -864,6 +873,12 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
864
873
|
# Honest capability report: trigram FTS5 keyword index, or
|
|
865
874
|
# LIKE-scan fallback when this SQLite build lacks it.
|
|
866
875
|
"fts_enabled": bool(getattr(self, "_fts_enabled", False)),
|
|
876
|
+
"engine": storage_capabilities,
|
|
877
|
+
"vector_search_backend": (
|
|
878
|
+
storage_capabilities.get("vector_backend")
|
|
879
|
+
if isinstance(storage_capabilities, dict)
|
|
880
|
+
else "bruteforce-cosine"
|
|
881
|
+
),
|
|
867
882
|
},
|
|
868
883
|
"source_items": len(source_items),
|
|
869
884
|
"indexed_items": sum(vector_counts.values()),
|