ltcai 4.2.0 → 4.3.1

package/lattice_brain/archive.py

@@ -1,8 +1,16 @@
-"""Encrypted .latticebrain archive support."""
+"""Encrypted .latticebrain archive support.
+
+The archive is intentionally self-contained and local-only: the encrypted
+payload holds the SQLite brain, blob store, portable JSON state, workspace
+export bundles when present, and public metadata needed to inspect/verify/
+restore on another machine without contacting a service.
+"""
 
 from __future__ import annotations
 
 import base64
+import hashlib
+import io
 import json
 import os
 import shutil
@@ -10,8 +18,8 @@ import tempfile
 import zipfile
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from pathlib import Path
-from typing import Dict, Optional
+from pathlib import Path, PurePosixPath
+from typing import Any, Dict, Iterable, List, Optional
 
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
@@ -20,8 +28,22 @@ from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
 
 ARCHIVE_FORMAT = "latticebrain.encrypted"
-ARCHIVE_VERSION = 1
+ARCHIVE_VERSION = 2
 KDF_ITERATIONS = 390_000
+PORTABLE_DATA_FILES = (
+    "users.json",
+    "chat_history.json",
+    "workspace_os.json",
+    "vpc_config.json",
+    "mcp_installs.json",
+    "audit_log.json",
+    "sso_config.json",
+    "hooks.json",
+    "invitations.json",
+    "agent_registry.json",
+    "brain_peers.json",
+)
+PORTABLE_EXPORT_SUFFIXES = (".json", ".zip")
 
 
 def _now() -> str:
@@ -40,10 +62,43 @@ def _derive_key(passphrase: str, salt: bytes) -> bytes:
     return kdf.derive(passphrase.encode("utf-8"))
 
 
+def _sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for block in iter(lambda: fh.read(65536), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def _safe_json(value: Any) -> Any:
+    try:
+        json.dumps(value)
+        return value
+    except TypeError:
+        if isinstance(value, dict):
+            return {str(k): _safe_json(v) for k, v in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [_safe_json(v) for v in value]
+        return str(value)
+
+
+def _assert_safe_member(name: str) -> PurePosixPath:
+    path = PurePosixPath(name)
+    if path.is_absolute() or ".." in path.parts:
+        raise ValueError(f"Archive payload contains unsafe path: {name}")
+    return path
+
+
 @dataclass(frozen=True)
 class BrainArchivePaths:
     db_path: Path
     blob_dir: Optional[Path] = None
+    data_dir: Optional[Path] = None
+    metadata: Optional[Dict[str, Any]] = None
 
 
 class EncryptedBrainArchive:
@@ -52,6 +107,67 @@ class EncryptedBrainArchive:
     def __init__(self, paths: BrainArchivePaths) -> None:
         self.paths = paths
 
+    def _iter_payload_files(self) -> Iterable[tuple[Path, str]]:
+        yield Path(self.paths.db_path), "knowledge_graph.sqlite"
+        if self.paths.blob_dir and self.paths.blob_dir.exists():
+            blob_root = Path(self.paths.blob_dir)
+            for file in sorted(blob_root.rglob("*")):
+                if file.is_file():
+                    yield file, f"blobs/{file.relative_to(blob_root).as_posix()}"
+        if self.paths.data_dir and Path(self.paths.data_dir).exists():
+            data_root = Path(self.paths.data_dir)
+            for name in PORTABLE_DATA_FILES:
+                file = data_root / name
+                if file.is_file():
+                    yield file, f"data/{name}"
+            exports = data_root / "workspace_exports"
+            if exports.exists():
+                for file in sorted(exports.rglob("*")):
+                    if not file.is_file():
+                        continue
+                    if file.suffix == ".latticebrain":
+                        continue
+                    if file.suffix not in PORTABLE_EXPORT_SUFFIXES:
+                        continue
+                    yield file, f"workspace_exports/{file.relative_to(exports).as_posix()}"
+
+    def _build_manifest(self, entries: List[Dict[str, Any]]) -> Dict[str, Any]:
+        metadata = _safe_json(self.paths.metadata or {})
+        return {
+            "format": "latticebrain.payload",
+            "format_version": ARCHIVE_VERSION,
+            "created_at": _now(),
+            "sections": {
+                "graph": any(item["path"] == "knowledge_graph.sqlite" for item in entries),
+                "blobs": any(item["path"].startswith("blobs/") for item in entries),
+                "workspace_state": any(item["path"].startswith("data/") for item in entries),
+                "signed_bundles": any(item["path"].startswith("workspace_exports/") for item in entries),
+            },
+            "metadata": metadata,
+            "storage": metadata.get("storage") or {},
+            "device_identity": metadata.get("device_identity") or {},
+            "provenance": metadata.get("provenance") or {},
+            "entries": entries,
+        }
+
+    def _payload_zip_bytes(self) -> tuple[bytes, Dict[str, Any]]:
+        entries: List[Dict[str, Any]] = []
+        with tempfile.TemporaryDirectory() as tmp_s:
+            payload = Path(tmp_s) / "payload.zip"
+            with zipfile.ZipFile(payload, "w", zipfile.ZIP_DEFLATED) as zf:
+                for src, arcname in self._iter_payload_files():
+                    _assert_safe_member(arcname)
+                    zf.write(src, arcname)
+                    entries.append({
+                        "path": arcname,
+                        "bytes": src.stat().st_size,
+                        "sha256": _sha256_file(src),
+                    })
+                manifest = self._build_manifest(entries)
+                manifest_bytes = json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True).encode("utf-8")
+                zf.writestr("manifest.json", manifest_bytes)
+            return payload.read_bytes(), manifest
+
     def create(self, destination: Path, *, passphrase: str) -> Dict[str, object]:
         dest = Path(destination)
         if dest.suffix != ".latticebrain":
@@ -59,59 +175,226 @@ class EncryptedBrainArchive:
         if not self.paths.db_path.exists():
             raise FileNotFoundError(f"Brain database not found: {self.paths.db_path}")
         dest.parent.mkdir(parents=True, exist_ok=True)
-        with tempfile.TemporaryDirectory() as tmp_s:
-            tmp = Path(tmp_s)
-            payload = tmp / "payload.zip"
-            with zipfile.ZipFile(payload, "w", zipfile.ZIP_DEFLATED) as zf:
-                zf.write(self.paths.db_path, "knowledge_graph.sqlite")
-                if self.paths.blob_dir and self.paths.blob_dir.exists():
-                    for file in self.paths.blob_dir.rglob("*"):
-                        if file.is_file():
-                            zf.write(file, f"blobs/{file.relative_to(self.paths.blob_dir)}")
-            salt = os.urandom(16)
-            nonce = os.urandom(12)
-            key = _derive_key(passphrase, salt)
-            ciphertext = AESGCM(key).encrypt(nonce, payload.read_bytes(), None)
-            envelope = {
-                "format": ARCHIVE_FORMAT,
-                "format_version": ARCHIVE_VERSION,
-                "created_at": _now(),
-                "kdf": {
-                    "name": "PBKDF2HMAC-SHA256",
-                    "iterations": KDF_ITERATIONS,
-                    "salt": base64.b64encode(salt).decode("ascii"),
-                },
-                "cipher": {
-                    "name": "AES-256-GCM",
-                    "nonce": base64.b64encode(nonce).decode("ascii"),
-                },
-                "payload": base64.b64encode(ciphertext).decode("ascii"),
-            }
-            dest.write_text(json.dumps(envelope, indent=2), encoding="utf-8")
-        return {"path": str(dest), "bytes": dest.stat().st_size, "encrypted": True}
+        payload_bytes, manifest = self._payload_zip_bytes()
+        salt = os.urandom(16)
+        nonce = os.urandom(12)
+        key = _derive_key(passphrase, salt)
+        ciphertext = AESGCM(key).encrypt(nonce, payload_bytes, None)
+        envelope = {
+            "format": ARCHIVE_FORMAT,
+            "format_version": ARCHIVE_VERSION,
+            "created_at": _now(),
+            "kdf": {
+                "name": "PBKDF2HMAC-SHA256",
+                "iterations": KDF_ITERATIONS,
+                "salt": base64.b64encode(salt).decode("ascii"),
+            },
+            "cipher": {
+                "name": "AES-256-GCM",
+                "nonce": base64.b64encode(nonce).decode("ascii"),
+            },
+            "payload_sha256": _sha256_bytes(payload_bytes),
+            "manifest_summary": {
+                "format_version": manifest["format_version"],
+                "created_at": manifest["created_at"],
+                "sections": manifest["sections"],
+                "storage": manifest.get("storage") or {},
+                "device_identity": manifest.get("device_identity") or {},
+            },
+            "payload": base64.b64encode(ciphertext).decode("ascii"),
+        }
+        dest.write_text(json.dumps(envelope, indent=2), encoding="utf-8")
+        return {
+            "path": str(dest),
+            "bytes": dest.stat().st_size,
+            "encrypted": True,
+            "format_version": ARCHIVE_VERSION,
+            "manifest": {
+                "sections": manifest["sections"],
+                "storage": manifest.get("storage") or {},
+                "entries": len(manifest["entries"]),
+            },
+        }
 
-    def restore(self, source: Path, *, passphrase: str, target: BrainArchivePaths) -> Dict[str, object]:
+    def _load_envelope(self, source: Path) -> Dict[str, Any]:
         src = Path(source)
         if not src.exists():
             raise FileNotFoundError(f"Brain archive not found: {src}")
-        envelope = json.loads(src.read_text(encoding="utf-8"))
+        try:
+            envelope = json.loads(src.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            raise ValueError("Archive envelope is not valid JSON.") from exc
         if envelope.get("format") != ARCHIVE_FORMAT:
             raise ValueError("Not a .latticebrain encrypted archive.")
-        salt = base64.b64decode(envelope["kdf"]["salt"])
-        nonce = base64.b64decode(envelope["cipher"]["nonce"])
-        ciphertext = base64.b64decode(envelope["payload"])
+        version = int(envelope.get("format_version") or 0)
+        if version < 1:
+            raise ValueError("Archive format version is missing or invalid.")
+        if version > ARCHIVE_VERSION:
+            raise ValueError(
+                f"Archive format version {version} is newer than supported version {ARCHIVE_VERSION}."
+            )
+        return envelope
+
+    def inspect(self, source: Path, *, passphrase: Optional[str] = None) -> Dict[str, Any]:
+        envelope = self._load_envelope(source)
+        summary = {
+            "valid_envelope": True,
+            "encrypted": True,
+            "format": envelope.get("format"),
+            "format_version": envelope.get("format_version"),
+            "created_at": envelope.get("created_at"),
+            "cipher": (envelope.get("cipher") or {}).get("name"),
+            "kdf": {
+                "name": (envelope.get("kdf") or {}).get("name"),
+                "iterations": (envelope.get("kdf") or {}).get("iterations"),
+            },
+            "manifest_summary": envelope.get("manifest_summary") or {},
+        }
+        if passphrase:
+            verified = self.verify(source, passphrase=passphrase)
+            summary["verified"] = verified["ok"]
+            summary["manifest"] = verified.get("manifest")
+            summary["errors"] = verified.get("errors", [])
+        return summary
+
+    def _decrypt_payload(self, envelope: Dict[str, Any], passphrase: str) -> bytes:
+        try:
+            salt = base64.b64decode(envelope["kdf"]["salt"])
+            nonce = base64.b64decode(envelope["cipher"]["nonce"])
+            ciphertext = base64.b64decode(envelope["payload"])
+        except Exception as exc:
+            raise ValueError("Archive envelope is missing encryption metadata.") from exc
         key = _derive_key(passphrase, salt)
         try:
-            plaintext = AESGCM(key).decrypt(nonce, ciphertext, None)
+            payload = AESGCM(key).decrypt(nonce, ciphertext, None)
         except InvalidTag as exc:
             raise ValueError("Archive decryption failed; passphrase or archive data is invalid.") from exc
+        expected = envelope.get("payload_sha256")
+        if expected and _sha256_bytes(payload) != expected:
+            raise ValueError("Archive payload integrity check failed (sha256 mismatch).")
+        return payload
+
+    def _read_payload(self, payload: bytes) -> tuple[Dict[str, Any], Dict[str, bytes]]:
+        try:
+            with zipfile.ZipFile(io.BytesIO(payload)) as zf:
+                bad_member = zf.testzip()
+                if bad_member:
+                    raise ValueError(f"Archive payload member is corrupt: {bad_member}")
+                names = zf.namelist()
+                for name in names:
+                    _assert_safe_member(name)
+                raw = {name: zf.read(name) for name in names if not name.endswith("/")}
+        except zipfile.BadZipFile as exc:
+            raise ValueError("Archive payload is not a valid ZIP file.") from exc
+        if "manifest.json" in raw:
+            manifest = json.loads(raw["manifest.json"].decode("utf-8"))
+        else:
+            manifest = {
+                "format": "latticebrain.payload",
+                "format_version": 1,
+                "created_at": None,
+                "sections": {
+                    "graph": "knowledge_graph.sqlite" in raw,
+                    "blobs": any(name.startswith("blobs/") for name in raw),
+                    "workspace_state": False,
+                    "signed_bundles": False,
+                },
+                "metadata": {},
+                "storage": {},
+                "device_identity": {},
+                "provenance": {},
+                "entries": [
+                    {"path": name, "bytes": len(data), "sha256": _sha256_bytes(data)}
+                    for name, data in raw.items()
+                    if name != "manifest.json"
+                ],
+            }
+        version = int(manifest.get("format_version") or 0)
+        if version < 1:
+            raise ValueError("Archive manifest format version is missing or invalid.")
+        if version > ARCHIVE_VERSION:
+            raise ValueError(
+                f"Archive manifest version {version} is newer than supported version {ARCHIVE_VERSION}."
+            )
+        return manifest, raw
+
+    def verify(self, source: Path, *, passphrase: str) -> Dict[str, Any]:
+        try:
+            envelope = self._load_envelope(source)
+            payload = self._decrypt_payload(envelope, passphrase)
+            manifest, raw = self._read_payload(payload)
+            if "knowledge_graph.sqlite" not in raw:
+                raise ValueError("Archive payload is missing knowledge_graph.sqlite.")
+            missing: List[str] = []
+            mismatched: List[str] = []
+            for entry in manifest.get("entries") or []:
+                name = str(entry.get("path") or "")
+                if not name or name == "manifest.json":
+                    continue
+                data = raw.get(name)
+                if data is None:
+                    missing.append(name)
+                    continue
+                if entry.get("sha256") and _sha256_bytes(data) != entry["sha256"]:
+                    mismatched.append(name)
+            if missing or mismatched:
+                raise ValueError(
+                    "Archive manifest integrity check failed "
+                    f"(missing={missing}, mismatched={mismatched})."
+                )
+            return {
+                "ok": True,
+                "encrypted": True,
+                "format_version": envelope.get("format_version"),
+                "manifest": manifest,
+                "entries": len([name for name in raw if name != "manifest.json"]),
+                "errors": [],
+            }
+        except (ValueError, FileNotFoundError) as exc:
+            return {"ok": False, "encrypted": True, "errors": [str(exc)]}
+
+    def restore(
+        self,
+        source: Path,
+        *,
+        passphrase: str,
+        target: BrainArchivePaths,
+        dry_run: bool = False,
+        confirm: bool = False,
+    ) -> Dict[str, object]:
+        if not dry_run and not confirm:
+            raise ValueError("Explicit confirmation is required before restoring a .latticebrain archive.")
+        envelope = self._load_envelope(source)
+        payload = self._decrypt_payload(envelope, passphrase)
+        manifest, raw = self._read_payload(payload)
+        verified = self.verify(source, passphrase=passphrase)
+        if not verified["ok"]:
+            raise ValueError("; ".join(verified.get("errors") or ["Archive verification failed."]))
+        planned = {
+            "database": str(target.db_path),
+            "blobs": str(target.blob_dir) if target.blob_dir else None,
+            "data_dir": str(target.data_dir) if target.data_dir else None,
+            "entries": len([name for name in raw if name != "manifest.json"]),
+            "sections": manifest.get("sections") or {},
+        }
+        if dry_run:
+            return {
+                "restored": False,
+                "dry_run": True,
+                "verified": True,
+                "planned": planned,
+                "manifest": manifest,
+            }
         with tempfile.TemporaryDirectory() as tmp_s:
             tmp = Path(tmp_s)
-            payload = tmp / "payload.zip"
-            payload.write_bytes(plaintext)
-            with zipfile.ZipFile(payload) as zf:
-                zf.extractall(tmp / "out")
-            db_src = tmp / "out" / "knowledge_graph.sqlite"
+            out = tmp / "out"
+            out.mkdir()
+            for name, data in raw.items():
+                _assert_safe_member(name)
+                dest = out / name
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                dest.write_bytes(data)
+            db_src = out / "knowledge_graph.sqlite"
             if not db_src.exists():
                 raise ValueError("Archive payload is missing knowledge_graph.sqlite.")
             target.db_path.parent.mkdir(parents=True, exist_ok=True)
@@ -119,7 +402,7 @@ class EncryptedBrainArchive:
                 if sibling.exists():
                     sibling.unlink()
             shutil.copyfile(db_src, target.db_path)
-            blobs_src = tmp / "out" / "blobs"
+            blobs_src = out / "blobs"
             if target.blob_dir:
                 if target.blob_dir.exists():
                     shutil.rmtree(target.blob_dir)
@@ -127,7 +410,37 @@ class EncryptedBrainArchive:
                     shutil.copytree(blobs_src, target.blob_dir)
                 else:
                     target.blob_dir.mkdir(parents=True, exist_ok=True)
-        return {"restored": True, "path": str(target.db_path), "encrypted": True}
+            if target.data_dir:
+                data_src = out / "data"
+                exports_src = out / "workspace_exports"
+                target_data = Path(target.data_dir)
+                target_data.mkdir(parents=True, exist_ok=True)
+                if data_src.exists():
+                    for file in sorted(data_src.rglob("*")):
+                        if file.is_file():
+                            rel = file.relative_to(data_src)
+                            if rel.as_posix() not in PORTABLE_DATA_FILES:
+                                continue
+                            dest = target_data / rel
+                            dest.parent.mkdir(parents=True, exist_ok=True)
+                            shutil.copyfile(file, dest)
+                if exports_src.exists():
+                    exports_dest = target_data / "workspace_exports"
+                    exports_dest.mkdir(parents=True, exist_ok=True)
+                    for file in sorted(exports_src.rglob("*")):
+                        if file.is_file():
+                            rel = file.relative_to(exports_src)
+                            dest = exports_dest / rel
+                            dest.parent.mkdir(parents=True, exist_ok=True)
+                            shutil.copyfile(file, dest)
+        return {
+            "restored": True,
+            "dry_run": False,
+            "path": str(target.db_path),
+            "encrypted": True,
+            "verified": True,
+            "manifest": manifest,
+        }
 
 
 __all__ = ["BrainArchivePaths", "EncryptedBrainArchive"]

package/lattice_brain/storage/sqlite.py

@@ -77,7 +77,12 @@ class SQLiteEngine(StorageEngine):
                 backup_restore=True,
                 migrations=True,
                 encrypted_archives=True,
-                metadata={"db_path": str(self.db_path), "sqlite_vec_loaded": False},
+                metadata={
+                    "db_path": str(self.db_path),
+                    "sqlite_vec_loaded": False,
+                    "vector_mode": "fallback",
+                    "honest_fallback": "sqlite-vec has not been probed yet; vector search uses the real brute-force cosine fallback until sqlite-vec is loaded.",
+                },
             )
         # Probe on demand so status is accurate even before the graph opens.
         try:
@@ -94,7 +99,11 @@ class SQLiteEngine(StorageEngine):
         return StorageCapabilities(
             engine=self.name,
             available=True,
-            reason=None if self._sqlite_vec_loaded else self._sqlite_vec_reason,
+            reason=None if self._sqlite_vec_loaded else (
+                f"{self._sqlite_vec_reason}; using real brute-force cosine fallback, not sqlite-vec ANN"
+                if self._sqlite_vec_reason
+                else "sqlite-vec unavailable; using real brute-force cosine fallback, not sqlite-vec ANN"
+            ),
             vector_backend=vector_backend,
             vector_available=True,
             backup_restore=True,
@@ -103,6 +112,10 @@ class SQLiteEngine(StorageEngine):
             metadata={
                 "db_path": str(self.db_path),
                 "sqlite_vec_loaded": self._sqlite_vec_loaded,
+                "sqlite_vec_ann_available": self._sqlite_vec_loaded,
+                "vector_mode": "sqlite-vec" if self._sqlite_vec_loaded else "fallback",
+                "degraded": not self._sqlite_vec_loaded,
+                "honest_fallback": None if self._sqlite_vec_loaded else "Vector search is available through the deterministic brute-force cosine backend. sqlite-vec ANN is unavailable.",
             },
         )
 

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "4.2.0"
+__version__ = "4.3.1"

package/latticeai/api/admin.py

@@ -56,6 +56,7 @@ def create_admin_router(
     invite_gate_enabled: bool,
     default_port: int,
     policy_matrix: Optional[Callable[[], List[Dict[str, object]]]] = None,
+    product_hardening_status: Optional[Callable[[], Dict[str, object]]] = None,
 ) -> APIRouter:
     router = APIRouter()
 
@@ -155,6 +156,16 @@ def create_admin_router(
             ]
         }
 
+    @router.get("/admin/product-hardening")
+    async def admin_product_hardening(request: Request):
+        require_admin(request)
+        if product_hardening_status is None:
+            return {
+                "available": False,
+                "reason": "Product hardening status provider is not configured.",
+            }
+        return product_hardening_status()
+
     @router.get("/vpc/status")
     async def vpc_status(request: Request):
         require_user(request)

package/latticeai/api/agents.py

@@ -46,7 +46,7 @@ def create_agents_router(
     run_executor: Any = None,
 ) -> APIRouter:
     from latticeai.core.multi_agent import AGENT_ROLES, ROLE_AGENT_IDS
-    from latticeai.services.agent_runtime import AgentRuntime
+    from latticeai.services.agent_runtime import AgentRuntime, AgentRuntimeUnavailable
 
     # Single AgentRuntime boundary: the router (and via it, the frontend) talks
     # to this façade instead of reaching into the orchestrator/store directly.
@@ -186,6 +186,8 @@ def create_agents_router(
             )
         except ValueError as exc:
             raise HTTPException(status_code=400, detail=str(exc)) from exc
+        except AgentRuntimeUnavailable as exc:
+            raise HTTPException(status_code=409, detail=str(exc)) from exc
         except PermissionError as exc:
             # A pre_run hook gated this run (e.g. a policy/permission hook).
             raise HTTPException(status_code=403, detail=str(exc)) from exc