ltcai 11.2.0 → 11.4.0

package/lattice_brain/portability/_contract.py

@@ -0,0 +1,42 @@
+"""The seam the two portability mixins share.
+
+``KGPortabilityService`` is assembled from :class:`KGPortabilitySharingMixin`
+and :class:`KGPortabilityBackupMixin`. Both read constructor state they do not
+own and both call the availability guard the service defines. That contract
+existed as an unwritten convention; this module writes it down.
+
+Typing-only, exactly like :mod:`lattice_brain.graph._kg_contract`: at runtime
+the mixins alias it to ``object``, so the MRO and every method resolution stay
+byte-for-byte what the single-file class had. Adding a cross-mixin call without
+declaring it here is a type error.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+class PortabilityCore:
+    """What either portability mixin may assume about ``self``.
+
+    Never instantiated and never inherited at runtime — see the module
+    docstring. Members are declared, not implemented: the implementation lives
+    in ``service.py`` (state and availability) or in whichever mixin owns it.
+    """
+
+    # ── State owned by KGPortabilityService.__init__ ─────────────────────────
+    _kg: Any
+    _data_dir: Path
+    _enable: bool
+    _exports_dir: Path
+    _identity: Any
+    _recipient: Any
+    _recipient_loaded: bool
+
+    # ── service.py: the availability guard every write door calls ────────────
+    def available(self) -> bool:
+        raise NotImplementedError
+
+    def _require(self) -> None:
+        raise NotImplementedError

package/lattice_brain/portability/backups.py

@@ -0,0 +1,338 @@
+"""Binary backup/restore, the encrypted archive, and the storage status reads.
+
+The half of :class:`~lattice_brain.portability.KGPortabilityService` that moves
+*files*: a ZIP snapshot of the database plus blobs, the passphrase-encrypted
+``.latticebrain`` archive, and the status/migration surfaces that sit on top of
+them (a Postgres migration refuses to start without a verified backup).
+
+Integrity hashing (``_sha256_file``) and the Postgres migrator are looked up as
+this module's globals, so a test standing in for either patches *this* module.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from ..archive import BrainArchivePaths, EncryptedBrainArchive
+from ..storage import (
+    DockerPostgresWizard,
+    PostgresEngine,
+    SQLiteToPostgresMigrator,
+)
+from ..utils import sha256_file as _sha256_file
+from ..utils import utc_now_iso
+from ._contract import PortabilityCore as _Core
+from .constants import BACKUP_FORMAT, FORMAT_VERSION, _stamp
+from .fsops import (
+    _pre_restore_backup_dir,
+    _replace_sqlite_atomically,
+    _replace_tree_with_backup,
+    _rollback_sqlite_from_backup,
+    _safe_zip_names,
+)
+
+
+class KGPortabilityBackupMixin(_Core):
+    """Backup / restore / storage status. Mixed into ``KGPortabilityService``."""
+
+    # ── binary backup / restore ──────────────────────────────────────────────
+    def backup(self, dest_path=None) -> Dict[str, Any]:
+        self._require()
+        self._exports_dir.mkdir(parents=True, exist_ok=True)
+        dest = Path(dest_path) if dest_path else self._exports_dir / f"kg-backup-{_stamp()}.zip"
+        with tempfile.TemporaryDirectory() as tmp_s:
+            tmp = Path(tmp_s)
+            db_copy = tmp / "knowledge_graph.sqlite"
+            self._kg.backup_database(db_copy)
+            manifest = {
+                "format": BACKUP_FORMAT,
+                "format_version": FORMAT_VERSION,
+                **self._kg.schema_versions(),
+                "created_at": utc_now_iso(),
+                "db_sha256": _sha256_file(db_copy),
+                "has_blobs": Path(self._kg.blob_dir).exists(),
+            }
+            with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as zf:
+                zf.write(db_copy, "knowledge_graph.sqlite")
+                zf.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
+                blob_dir = Path(self._kg.blob_dir)
+                if blob_dir.exists():
+                    for f in blob_dir.rglob("*"):
+                        if f.is_file():
+                            zf.write(f, f"blobs/{f.relative_to(blob_dir)}")
+        return {"path": str(dest), "bytes": dest.stat().st_size, "manifest": manifest}
+
+    def restore(
+        self,
+        archive_path,
+        *,
+        verify: bool = True,
+        dry_run: bool = False,
+        confirm: bool = False,
+    ) -> Dict[str, Any]:
+        self._require()
+        archive = Path(archive_path)
+        if not archive.exists():
+            raise FileNotFoundError(f"Backup archive not found: {archive}")
+        if not dry_run and not confirm:
+            raise ValueError("Explicit confirmation is required before restoring a Knowledge Graph backup.")
+        with zipfile.ZipFile(archive) as zf:
+            names = zf.namelist()
+            _safe_zip_names(names)
+            if "knowledge_graph.sqlite" not in names:
+                raise ValueError("Archive is missing knowledge_graph.sqlite.")
+            manifest = json.loads(zf.read("manifest.json")) if "manifest.json" in names else {}
+            with tempfile.TemporaryDirectory() as tmp_s:
+                tmp = Path(tmp_s)
+                zf.extractall(tmp)
+                db_src = tmp / "knowledge_graph.sqlite"
+                if verify and manifest.get("db_sha256"):
+                    if _sha256_file(db_src) != manifest["db_sha256"]:
+                        raise ValueError("Backup integrity check failed (db sha256 mismatch).")
+                if dry_run:
+                    return {
+                        "restored": False,
+                        "dry_run": True,
+                        "verified": True,
+                        "manifest": manifest,
+                        "planned": {
+                            "database": str(self._kg.db_path),
+                            "blobs": str(self._kg.blob_dir),
+                            "archive": str(archive),
+                        },
+                    }
+                db_dest = Path(self._kg.db_path)
+                blob_dest = Path(self._kg.blob_dir)
+                backup_dir = _pre_restore_backup_dir(db_dest)
+                try:
+                    _replace_sqlite_atomically(db_src, db_dest, backup_dir)
+                    blob_src = tmp / "blobs"
+                    _replace_tree_with_backup(blob_src if blob_src.exists() else None, blob_dest, backup_dir)
+                except Exception:
+                    # The rollback is best-effort recovery; an I/O slip inside
+                    # it must never mask the restore failure being reported.
+                    try:
+                        _rollback_sqlite_from_backup(db_dest, backup_dir)
+                    except OSError as rollback_exc:
+                        logging.warning(
+                            "pre-restore rollback incomplete (backup kept at %s): %s",
+                            backup_dir, rollback_exc,
+                        )
+                    raise
+        stats = self._kg.stats()
+        return {
+            "restored": True,
+            "manifest": manifest,
+            "pre_restore_backup": str(backup_dir),
+            "nodes": sum(stats.get("nodes", {}).values()),
+        }
+
+    def verify_backup(self, archive_path) -> Dict[str, Any]:
+        archive = Path(archive_path)
+        if not archive.exists():
+            return {"ok": False, "path": str(archive), "errors": [f"Backup archive not found: {archive}"]}
+        try:
+            with zipfile.ZipFile(archive) as zf:
+                names = zf.namelist()
+                _safe_zip_names(names)
+                if "knowledge_graph.sqlite" not in names:
+                    raise ValueError("Archive is missing knowledge_graph.sqlite.")
+                manifest = json.loads(zf.read("manifest.json")) if "manifest.json" in names else {}
+                with tempfile.TemporaryDirectory() as tmp_s:
+                    tmp = Path(tmp_s)
+                    zf.extract("knowledge_graph.sqlite", tmp)
+                    db_src = tmp / "knowledge_graph.sqlite"
+                    if manifest.get("db_sha256") and _sha256_file(db_src) != manifest["db_sha256"]:
+                        raise ValueError("Backup integrity check failed (db sha256 mismatch).")
+            return {"ok": True, "path": str(archive), "manifest": manifest, "errors": []}
+        except (ValueError, zipfile.BadZipFile, OSError, json.JSONDecodeError) as exc:
+            return {"ok": False, "path": str(archive), "errors": [str(exc)]}
+
+    # ── encrypted .latticebrain archive ───────────────────────────────────
+    def encrypted_archive(self, dest_path=None, *, passphrase: str) -> Dict[str, Any]:
+        self._require()
+        self._exports_dir.mkdir(parents=True, exist_ok=True)
+        dest = Path(dest_path) if dest_path else self._exports_dir / f"brain-{_stamp()}.latticebrain"
+        metadata = {
+            "storage": self.storage_status().get("active", {}),
+            "snapshot": self.snapshot_metadata(),
+            "device_identity": self._identity.describe() if self._identity is not None else {},
+            "provenance": {"exported_at": utc_now_iso(), "source": "kg-portability"},
+        }
+        archive = EncryptedBrainArchive(
+            BrainArchivePaths(
+                db_path=Path(self._kg.db_path),
+                blob_dir=Path(self._kg.blob_dir),
+                data_dir=self._data_dir,
+                metadata=metadata,
+            )
+        )
+        return archive.create(dest, passphrase=passphrase)
+
+    def inspect_encrypted_archive(self, archive_path, *, passphrase: Optional[str] = None) -> Dict[str, Any]:
+        archive = EncryptedBrainArchive(
+            BrainArchivePaths(
+                db_path=Path(self._kg.db_path),
+                blob_dir=Path(self._kg.blob_dir),
+                data_dir=self._data_dir,
+            )
+        )
+        return archive.inspect(Path(archive_path), passphrase=passphrase)
+
+    def verify_encrypted_archive(self, archive_path, *, passphrase: str) -> Dict[str, Any]:
+        archive = EncryptedBrainArchive(
+            BrainArchivePaths(
+                db_path=Path(self._kg.db_path),
+                blob_dir=Path(self._kg.blob_dir),
+                data_dir=self._data_dir,
+            )
+        )
+        return archive.verify(Path(archive_path), passphrase=passphrase)
+
+    def restore_encrypted_archive(
+        self,
+        archive_path,
+        *,
+        passphrase: str,
+        dry_run: bool = False,
+        confirm: bool = False,
+    ) -> Dict[str, Any]:
+        self._require()
+        archive = EncryptedBrainArchive(
+            BrainArchivePaths(
+                db_path=Path(self._kg.db_path),
+                blob_dir=Path(self._kg.blob_dir),
+                data_dir=self._data_dir,
+            )
+        )
+        return archive.restore(
+            Path(archive_path),
+            passphrase=passphrase,
+            target=BrainArchivePaths(
+                db_path=Path(self._kg.db_path),
+                blob_dir=Path(self._kg.blob_dir),
+                data_dir=self._data_dir,
+            ),
+            dry_run=dry_run,
+            confirm=confirm,
+        )
+
+    def import_encrypted_archive(
+        self,
+        archive_path,
+        *,
+        passphrase: str,
+        dry_run: bool = False,
+        confirm: bool = False,
+    ) -> Dict[str, Any]:
+        result = self.restore_encrypted_archive(
+            archive_path,
+            passphrase=passphrase,
+            dry_run=dry_run,
+            confirm=confirm,
+        )
+        result["operation"] = "import"
+        return result
+
+    # ── status surface ───────────────────────────────────────────────────────
+    def snapshot_metadata(self) -> Dict[str, Any]:
+        if not self.available():
+            return {"available": False}
+        return {
+            "available": True,
+            **self._kg.schema_versions(),
+            "stats": self._kg.stats(),
+            "provenance": self._kg.provenance_stats(),
+            "storage": (
+                self._kg.storage_engine.capabilities().as_dict()
+                if getattr(self._kg, "storage_engine", None) is not None
+                else {"engine": "sqlite", "available": True}
+            ),
+        }
+
+    def storage_status(self) -> Dict[str, Any]:
+        if not self.available():
+            return {"available": False}
+        return {
+            "available": True,
+            "active": (
+                self._kg.storage_engine.capabilities().as_dict()
+                if getattr(self._kg, "storage_engine", None) is not None
+                else {"engine": "sqlite", "available": True}
+            ),
+            "postgres": PostgresEngine("", schema="lattice_brain").capabilities().as_dict(),
+            "backup_health": self.backup_health(),
+        }
+
+    def backup_health(self) -> Dict[str, Any]:
+        self._exports_dir.mkdir(parents=True, exist_ok=True)
+        backups = sorted(
+            [
+                p for p in self._exports_dir.glob("*")
+                if p.is_file() and (p.suffix == ".zip" or p.suffix == ".latticebrain")
+            ],
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+        latest = backups[0] if backups else None
+        return {
+            "available": True,
+            "directory": str(self._exports_dir),
+            "count": len(backups),
+            "latest": str(latest) if latest else None,
+            "latest_bytes": latest.stat().st_size if latest else 0,
+            "encrypted_archives": sum(1 for p in backups if p.suffix == ".latticebrain"),
+            "zip_backups": sum(1 for p in backups if p.suffix == ".zip"),
+        }
+
+    def postgres_docker_setup(
+        self,
+        *,
+        consent: bool,
+        dry_run: bool = False,
+        port: int = 5432,
+    ) -> Dict[str, Any]:
+        wizard = DockerPostgresWizard(self._data_dir / "postgres", port=port)
+        return wizard.start(consent=consent, dry_run=dry_run)
+
+    def migrate_sqlite_to_postgres(
+        self,
+        *,
+        dsn: str,
+        schema: str = "lattice_brain",
+        dry_run: bool = True,
+    ) -> Dict[str, Any]:
+        self._require()
+        if not dsn:
+            raise ValueError("Postgres DSN is required for SQLite to Postgres migration.")
+        migrator = SQLiteToPostgresMigrator(
+            Path(self._kg.db_path),
+            PostgresEngine(dsn, schema=schema),
+        )
+        if dry_run:
+            return migrator.migrate(dry_run=True)
+        backup = self.backup()
+        verification = self.verify_backup(backup["path"])
+        if not verification.get("ok"):
+            raise RuntimeError(
+                "Pre-migration backup verification failed; Postgres migration was not started: "
+                + "; ".join(verification.get("errors") or [])
+            )
+        result = migrator.migrate(dry_run=False)
+        result["pre_migration_backup"] = {
+            "path": backup["path"],
+            "verified": True,
+            "manifest": backup.get("manifest"),
+        }
+        return result
+
+    def recent_ingestions(self, *, limit: int = 50, source_type: Optional[str] = None) -> Dict[str, Any]:
+        """Recent provenance records (newest first) for the ingestion-sources UI."""
+        if not self.available():
+            return {"items": [], "count": 0}
+        return self._kg.list_provenance(limit=limit, source_type=source_type)

package/lattice_brain/portability/bundles.py

@@ -0,0 +1,136 @@
+"""Pure shaping of a shared subgraph bundle: select, expand, redact, digest.
+
+None of these touch a store or a file. They decide *what* travels — which
+nodes the selection names, which neighbours a one-hop expansion may admit, and
+which sender-identifying fields are stripped on the way out — and they pin the
+result with the digest the signature covers.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any, Dict, List, Optional, Set
+
+from .constants import _NEIGHBOR_EXCLUDED_TYPES, _REDACTED_FIELDS
+
+
+def _canonical_digest(payload: Dict[str, Any]) -> str:
+    """sha256 over the canonical JSON of a bundle payload.
+
+    Pinned in the signed header so the Ed25519 signature covers the contents,
+    not just the manifest describing them.
+    """
+    canonical = json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
+    return hashlib.sha256(canonical).hexdigest()
+
+
+def _node_source_type(node: Dict[str, Any]) -> str:
+    try:
+        metadata = json.loads(node.get("metadata_json") or "{}")
+    except (TypeError, ValueError):
+        return ""
+    if not isinstance(metadata, dict):
+        return ""
+    return str(metadata.get("source_type") or "").strip().lower()
+
+
+def _select_node_ids(nodes: List[Dict[str, Any]], selection: Dict[str, Any]) -> Set[str]:
+    wanted_ids = set(selection["node_ids"])
+    wanted_types = set(selection["node_types"])
+    wanted_sources = set(selection["source_types"])
+    keep: Set[str] = set()
+    for node in nodes:
+        node_id = str(node.get("id"))
+        if node_id in wanted_ids:
+            keep.add(node_id)
+            continue
+        if wanted_types and str(node.get("type") or "").strip().lower() in wanted_types:
+            keep.add(node_id)
+            continue
+        if wanted_sources and _node_source_type(node) in wanted_sources:
+            keep.add(node_id)
+    return keep
+
+
+def _expand_neighbors(
+    keep: Set[str], edges: List[Dict[str, Any]], known: Dict[str, str]
+) -> Set[str]:
+    """One hop out from the selection, in both directions, within the scope.
+
+    ``Person`` and ``Source`` neighbours are never pulled in implicitly: a
+    Person node's label *is* an email address and a Source node's summary *is*
+    a local file path, so "share this decision and what it touches" would
+    quietly ship the sender's identity and directory layout. Naming one of
+    those node ids explicitly still exports it — that is a decision, not a
+    side effect.
+    """
+    expanded = set(keep)
+
+    def _admit(candidate: str) -> None:
+        if candidate in known and known[candidate] not in _NEIGHBOR_EXCLUDED_TYPES:
+            expanded.add(candidate)
+
+    for edge in edges:
+        source = str(edge.get("from_node"))
+        target = str(edge.get("to_node"))
+        if source in keep:
+            _admit(target)
+        if target in keep:
+            _admit(source)
+    return expanded
+
+
+def _strip_fields(raw_json: Optional[str]) -> Optional[str]:
+    if not raw_json:
+        return raw_json
+    try:
+        parsed = json.loads(raw_json)
+    except (TypeError, ValueError):
+        return raw_json
+    if not isinstance(parsed, dict):
+        return raw_json
+    cleaned = {k: v for k, v in parsed.items() if k not in _REDACTED_FIELDS}
+    return json.dumps(cleaned, ensure_ascii=False, sort_keys=True)
+
+
+def _redact_node(node: Dict[str, Any]) -> Dict[str, Any]:
+    redacted = dict(node)
+    redacted["metadata_json"] = _strip_fields(node.get("metadata_json"))
+    redacted["raw_json"] = _strip_fields(node.get("raw_json"))
+    return redacted
+
+
+def _redact_provenance_row(row: Dict[str, Any]) -> Dict[str, Any]:
+    redacted = dict(row)
+    redacted["owner"] = None
+    redacted["source_uri"] = None
+    redacted["permissions_json"] = "{}"
+    redacted["metadata_json"] = _strip_fields(row.get("metadata_json"))
+    return redacted
+
+
+def _scope_node(node: Dict[str, Any], workspace_id: str) -> Dict[str, Any]:
+    """Stamp the accepting workspace onto an incoming node.
+
+    Received knowledge belongs to the workspace that accepted it. Without this
+    the projection would file it as legacy-global — machine-shared — which is a
+    wider scope than the reviewer chose.
+    """
+    scoped = dict(node)
+    try:
+        metadata = json.loads(scoped.get("metadata_json") or "{}")
+    except (TypeError, ValueError):
+        metadata = {}
+    if not isinstance(metadata, dict):
+        metadata = {}
+    metadata["workspace_id"] = workspace_id
+    scoped["metadata_json"] = json.dumps(metadata, ensure_ascii=False, sort_keys=True)
+    return scoped
+
+
+def _shared_node_summary(node: Dict[str, Any], verdict: Dict[str, Any]) -> str:
+    body = str(node.get("summary") or "").strip()
+    origin = verdict.get("origin") or "unknown device"
+    prefix = f"[{origin}] "
+    return (prefix + body)[:500] if body else prefix.strip()

package/lattice_brain/portability/constants.py

@@ -0,0 +1,93 @@
+"""Formats, gates and the vocabulary the portability surfaces agree on.
+
+Everything here is a literal or a gate: no I/O, no graph, no service. Keeping
+it in one leaf module is what lets the file helpers, the sharing surface and
+the backup surface all import the same constants without importing each other.
+"""
+
+from __future__ import annotations
+
+from ..gates import FeatureGate
+from ..utils import utc_now_iso
+
+FORMAT = "latticeai.kg.export"
+FORMAT_VERSION = 1
+BACKUP_FORMAT = "latticeai.kg.backup"
+
+# ── selective subgraph share (v11.1.0 prototype, off by default) ─────────────
+#: Logical format of a *partial* bundle: a chosen slice of the graph plus its
+#: provenance, signed by this device. Deliberately distinct from ``FORMAT`` so
+#: the whole-graph importer can never be handed a subgraph by accident.
+SUBGRAPH_FORMAT = "latticeai.kg.subgraph"
+SUBGRAPH_FORMAT_VERSION = 1
+#: Encrypted on-disk envelope for the same bundle, reusing the ``.latticebrain``
+#: passphrase mechanism (PBKDF2-SHA256 + AES-256-GCM) from ``archive.py``.
+SUBGRAPH_ARCHIVE_FORMAT = "latticebrain.subgraph"
+#: Opt-in gate. Off unless the operator sets it: sharing knowledge with another
+#: Brain is a network act, and local-first means those are never on by default.
+BRAIN_NETWORK_ENV = "LATTICEAI_BRAIN_NETWORK"
+_TRUTHY = frozenset({"1", "true", "yes", "on"})
+#: How a bundle can be encrypted. ``passphrase`` needs a secret agreed out of
+#: band; ``recipient_public_key`` needs nothing secret to travel at all.
+ENCRYPTION_MODES = ("passphrase", "recipient_public_key")
+#: A received bundle becomes review items, never a merge. The cap keeps one
+#: peer from flooding the inbox; what it dropped is reported, not hidden.
+SUBGRAPH_PROPOSAL_CAP = 200
+#: Review-queue vocabulary. ``kg_change_digest`` is the existing source for
+#: "the graph would change"; the kind distinguishes shared-subgraph items.
+SUBGRAPH_REVIEW_SOURCE = "kg_change_digest"
+SUBGRAPH_REVIEW_KIND = "shared_subgraph_node"
+#: Mirrors ``services.review_queue.OPEN_STATUSES``. Duplicated as a literal
+#: rather than imported because ``lattice_brain`` must not depend on the
+#: ``latticeai`` service layer; the review sink is injected, not imported.
+_OPEN_REVIEW_STATUSES = frozenset({"pending", "snoozed"})
+
+BRAIN_NETWORK_DISABLED_DETAIL = (
+    "Brain Network sharing is off. It is opt-in by design: set "
+    f"{BRAIN_NETWORK_ENV}=1 to enable selective subgraph export and receipt."
+)
+
+
+#: The share gate, resolved when it is asked. The environment variable is still
+#: the default answer — an untouched install behaves exactly as it did — but a
+#: settings surface can now bind a resolver and move it without a restart.
+BRAIN_NETWORK_GATE = FeatureGate(
+    BRAIN_NETWORK_ENV,
+    default=False,
+    name="brain_network",
+    detail=BRAIN_NETWORK_DISABLED_DETAIL,
+)
+
+
+class BrainNetworkDisabled(PermissionError):
+    """Raised when a share surface is used while the opt-in flag is off."""
+
+    def __init__(self, detail: str = BRAIN_NETWORK_DISABLED_DETAIL) -> None:
+        super().__init__(detail)
+
+
+def brain_network_enabled() -> bool:
+    """True only when the operator explicitly opted in (default: off)."""
+    return BRAIN_NETWORK_GATE.enabled()
+
+
+def require_brain_network() -> None:
+    if not brain_network_enabled():
+        raise BrainNetworkDisabled()
+
+
+def _stamp() -> str:
+    return utc_now_iso().replace(":", "").replace("-", "").replace(".", "")[:15]
+
+
+#: Sender-identifying fields stripped from a shared bundle by default. The
+#: knowledge travels; "which account, on which machine, at which path" does not.
+_REDACTED_FIELDS = frozenset({"owner", "user_email", "source_uri", "permissions"})
+#: The signed payload's key set. One tuple so export and verify can never
+#: disagree about what the digest covers.
+_PAYLOAD_KEYS = (
+    "nodes", "chunk_nodes", "edges", "chunks", "knowledge_sources", "provenance",
+)
+#: Node types a one-hop expansion never admits — they describe the sender
+#: (identity, local paths), not the knowledge being shared.
+_NEIGHBOR_EXCLUDED_TYPES = frozenset({"Person", "Source"})