ltcai 4.3.3 → 4.4.0

package/lattice_brain/graph/documents.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphDocumentsMixin:
+    def _ingest_structure_nodes(
+        self,
+        conn: sqlite3.Connection,
+        file_id: str,
+        filename: str,
+        structure: Dict[str, Any],
+    ) -> None:
+        for slide in structure.get("slides") or []:
+            index = slide.get("index")
+            slide_id = f"slide:{_sha256_text(f'{file_id}:slide:{index}')[:24]}"
+            title = f"{filename} slide {index}"
+            summary = "\n".join(slide.get("texts") or [])[:800]
+            self._upsert_node(
+                conn, slide_id, "Slide", title, summary=summary, metadata=slide
+            )
+            self._upsert_edge(conn, file_id, slide_id, "has_slide")
+            for text in slide.get("texts") or []:
+                for topic in _topic_candidates(text, limit=4):
+                    topic_id = f"topic:{_slug(topic)}"
+                    self._upsert_node(
+                        conn,
+                        topic_id,
+                        "Topic",
+                        topic,
+                        metadata={"auto_extracted": True},
+                    )
+                    self._upsert_edge(conn, slide_id, topic_id, "discusses", weight=0.6)
+
+        for page in structure.get("pages") or []:
+            index = page.get("index")
+            page_id = f"page:{_sha256_text(f'{file_id}:page:{index}')[:24]}"
+            title = f"{filename} page {index}"
+            self._upsert_node(
+                conn,
+                page_id,
+                "Page",
+                title,
+                summary=page.get("preview") or "",
+                metadata=page,
+            )
+            self._upsert_edge(conn, file_id, page_id, "has_page")
+            for topic in _topic_candidates(page.get("preview") or "", limit=4):
+                topic_id = f"topic:{_slug(topic)}"
+                self._upsert_node(
+                    conn, topic_id, "Topic", topic, metadata={"auto_extracted": True}
+                )
+                self._upsert_edge(conn, page_id, topic_id, "discusses", weight=0.6)
+
+        for sheet in structure.get("sheets") or []:
+            sheet_title = sheet.get("title")
+            sheet_id = f"sheet:{_sha256_text(f'{file_id}:sheet:{sheet_title}')[:24]}"
+            self._upsert_node(
+                conn, sheet_id, "Sheet", f"{filename} / {sheet_title}", metadata=sheet
+            )
+            self._upsert_edge(conn, file_id, sheet_id, "has_sheet")
+
+        for image in structure.get("images") or []:
+            image_key = image.get("sha256") or _sha256_text(
+                json.dumps(image, ensure_ascii=False, sort_keys=True)
+            )
+            image_id = f"image:{str(image_key)[:24]}"
+            title_parts = [filename, "image"]
+            if image.get("page"):
+                title_parts.append(f"page {image.get('page')}")
+            if image.get("name"):
+                title_parts.append(str(image.get("name")).split("/")[-1])
+            self._upsert_node(
+                conn, image_id, "Image", " / ".join(title_parts), metadata=image
+            )
+            self._upsert_edge(conn, file_id, image_id, "contains_image")
+
+    def _document_structure(self, path: Path, ext: str) -> Dict[str, Any]:
+        try:
+            if ext == ".pptx":
+                return self._pptx_structure(path)
+            if ext == ".pdf":
+                return self._pdf_structure(path)
+            if ext == ".docx":
+                return self._docx_structure(path)
+            if ext == ".xlsx":
+                return self._xlsx_structure(path)
+        except Exception as exc:
+            return {"error": str(exc)}
+        return {}
+
+    def _pptx_structure(self, path: Path) -> Dict[str, Any]:
+        result: Dict[str, Any] = {"slides": [], "images": []}
+        try:
+            from PIL import Image
+            from pptx import Presentation
+
+            prs = Presentation(str(path))
+            for slide_index, slide in enumerate(prs.slides, start=1):
+                slide_info = {"index": slide_index, "shapes": [], "texts": []}
+                for shape_index, shape in enumerate(slide.shapes, start=1):
+                    shape_info = {
+                        "index": shape_index,
+                        "name": getattr(shape, "name", ""),
+                        "shape_type": str(getattr(shape, "shape_type", "")),
+                        "bbox": {
+                            "left": int(getattr(shape, "left", 0) or 0),
+                            "top": int(getattr(shape, "top", 0) or 0),
+                            "width": int(getattr(shape, "width", 0) or 0),
+                            "height": int(getattr(shape, "height", 0) or 0),
+                        },
+                    }
+                    if getattr(shape, "has_text_frame", False):
+                        text = shape.text_frame.text.strip()
+                        if text:
+                            shape_info["text"] = text[:1000]
+                            slide_info["texts"].append(text)
+                    slide_info["shapes"].append(shape_info)
+                result["slides"].append(slide_info)
+            with zipfile.ZipFile(path) as zf:
+                for name in zf.namelist():
+                    if not name.startswith("ppt/media/"):
+                        continue
+                    data = zf.read(name)
+                    image_info: Dict[str, Any] = {
+                        "name": name,
+                        "bytes": len(data),
+                        "sha256": _sha256_bytes(data),
+                    }
+                    try:
+                        from io import BytesIO
+
+                        with Image.open(BytesIO(data)) as img:
+                            image_info.update(
+                                {
+                                    "width": img.width,
+                                    "height": img.height,
+                                    "format": img.format,
+                                }
+                            )
+                    except Exception:
+                        pass
+                    result["images"].append(image_info)
+        except Exception as exc:
+            result["error"] = str(exc)
+        return result
+
+    def _pdf_structure(self, path: Path) -> Dict[str, Any]:
+        result: Dict[str, Any] = {"pages": [], "images": []}
+        try:
+            import pdfplumber
+
+            with pdfplumber.open(str(path)) as pdf:
+                metadata = dict(pdf.metadata or {})
+                result["metadata"] = {str(k): str(v) for k, v in metadata.items()}
+                for page_index, page in enumerate(pdf.pages, start=1):
+                    text = page.extract_text() or ""
+                    page_info = {
+                        "index": page_index,
+                        "width": float(page.width or 0),
+                        "height": float(page.height or 0),
+                        "chars": len(text),
+                        "preview": _clean_text(text)[:500],
+                        "image_count": len(page.images or []),
+                    }
+                    result["pages"].append(page_info)
+                    for image_index, image in enumerate(page.images or [], start=1):
+                        result["images"].append(
+                            {
+                                "page": page_index,
+                                "index": image_index,
+                                "name": image.get("name"),
+                                "width": image.get("width"),
+                                "height": image.get("height"),
+                                "bbox": {
+                                    "x0": image.get("x0"),
+                                    "top": image.get("top"),
+                                    "x1": image.get("x1"),
+                                    "bottom": image.get("bottom"),
+                                },
+                            }
+                        )
+        except Exception as exc:
+            result["error"] = str(exc)
+        return result
+
+    def _docx_structure(self, path: Path) -> Dict[str, Any]:
+        from docx import Document
+
+        doc = Document(str(path))
+        headings = []
+        paragraphs = 0
+        for p in doc.paragraphs:
+            text = p.text.strip()
+            if not text:
+                continue
+            paragraphs += 1
+            style = getattr(p.style, "name", "")
+            if style.lower().startswith("heading"):
+                headings.append({"style": style, "text": text[:240]})
+        return {
+            "paragraphs": paragraphs,
+            "headings": headings[:80],
+            "tables": len(doc.tables),
+        }
+
+    def _xlsx_structure(self, path: Path) -> Dict[str, Any]:
+        from openpyxl import load_workbook
+
+        wb = load_workbook(str(path), read_only=True, data_only=True)
+        sheets = []
+        for ws in wb.worksheets:
+            sheets.append(
+                {"title": ws.title, "max_row": ws.max_row, "max_column": ws.max_column}
+            )
+        return {"sheets": sheets}

package/lattice_brain/graph/identity.py

@@ -0,0 +1,175 @@
+"""Device identity — the sovereignty primitive.
+
+Every Lattice installation owns an Ed25519 keypair. Exports are signed by
+it, peers pair against its public key, and imported knowledge records which
+device it came from. The private key never leaves the machine: it lives in
+the OS keyring when one is available, otherwise in a 0600 file under the
+data directory (the storage backend is reported honestly).
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+    Ed25519PrivateKey,
+    Ed25519PublicKey,
+)
+
+_KEYRING_SERVICE = "lattice-ai-device-identity"
+_KEYRING_ENTRY = "ed25519-private-key"
+
+
+def _b64(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=")
+
+
+def _unb64(text: str) -> bytes:
+    padded = text + "=" * (-len(text) % 4)
+    return base64.urlsafe_b64decode(padded.encode("ascii"))
+
+
+def _keyring_opt_in() -> bool:
+    """Keyring storage is opt-in (LATTICEAI_DEVICE_KEY_KEYRING=1).
+
+    OS keychain access can block or prompt during startup/tests; the default
+    is a 0600 file under the data dir, and ``describe()`` reports which
+    backend holds the key — no silent security theater either way.
+    """
+    return os.getenv("LATTICEAI_DEVICE_KEY_KEYRING", "").strip() in {"1", "true", "yes"}
+
+
+class DeviceIdentity:
+    """Loads-or-creates the installation's Ed25519 keypair."""
+
+    def __init__(self, data_dir: Path, *, use_keyring: Optional[bool] = None):
+        if use_keyring is None:
+            use_keyring = _keyring_opt_in()
+        self._data_dir = Path(data_dir)
+        self._key_file = self._data_dir / "device_identity.key"
+        self._private: Ed25519PrivateKey
+        self.storage: str  # "keyring" | "file"
+        self._load_or_create(use_keyring)
+
+    # ── key material ───────────────────────────────────────────────────────
+    def _load_or_create(self, use_keyring: bool) -> None:
+        raw: Optional[bytes] = None
+        backend = "file"
+        if use_keyring:
+            try:
+                import keyring
+
+                stored = keyring.get_password(_KEYRING_SERVICE, _KEYRING_ENTRY)
+                if stored:
+                    raw = _unb64(stored)
+                    backend = "keyring"
+            except Exception as exc:
+                logging.debug("device identity: keyring unavailable (%s)", exc)
+        if raw is None and self._key_file.exists():
+            raw = _unb64(self._key_file.read_text().strip())
+            backend = "file"
+        if raw is None:
+            key = Ed25519PrivateKey.generate()
+            raw = key.private_bytes(
+                serialization.Encoding.Raw,
+                serialization.PrivateFormat.Raw,
+                serialization.NoEncryption(),
+            )
+            backend = self._persist_new(raw, use_keyring)
+        self._private = Ed25519PrivateKey.from_private_bytes(raw)
+        self.storage = backend
+
+    def _persist_new(self, raw: bytes, use_keyring: bool) -> str:
+        if use_keyring:
+            try:
+                import keyring
+
+                keyring.set_password(_KEYRING_SERVICE, _KEYRING_ENTRY, _b64(raw))
+                return "keyring"
+            except Exception as exc:
+                logging.debug("device identity: keyring store failed (%s); using file", exc)
+        self._data_dir.mkdir(parents=True, exist_ok=True)
+        self._key_file.write_text(_b64(raw))
+        os.chmod(self._key_file, 0o600)
+        return "file"
+
+    # ── public surface ─────────────────────────────────────────────────────
+    @property
+    def public_key_b64(self) -> str:
+        return _b64(
+            self._private.public_key().public_bytes(
+                serialization.Encoding.Raw, serialization.PublicFormat.Raw
+            )
+        )
+
+    @property
+    def fingerprint(self) -> str:
+        """Short human-comparable id: sha256 of the raw public key."""
+        raw = self._private.public_key().public_bytes(
+            serialization.Encoding.Raw, serialization.PublicFormat.Raw
+        )
+        digest = hashlib.sha256(raw).hexdigest()
+        return ":".join(digest[i : i + 4] for i in range(0, 16, 4))
+
+    def describe(self) -> Dict[str, Any]:
+        return {
+            "fingerprint": self.fingerprint,
+            "public_key": self.public_key_b64,
+            "algorithm": "ed25519",
+            "storage": self.storage,
+        }
+
+    # ── signing ────────────────────────────────────────────────────────────
+    def sign(self, payload: bytes) -> str:
+        return _b64(self._private.sign(payload))
+
+    def sign_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
+        """Detached signature over the canonical JSON of a manifest."""
+        canonical = json.dumps(manifest, sort_keys=True, ensure_ascii=False).encode("utf-8")
+        return {
+            "algorithm": "ed25519",
+            "public_key": self.public_key_b64,
+            "fingerprint": self.fingerprint,
+            "signature": self.sign(canonical),
+        }
+
+
+def fingerprint_of(public_key_b64: str) -> str:
+    """Human-comparable fingerprint of an Ed25519 public key.
+
+    Raises ValueError when the input is not a valid key — the pairing flow
+    uses this as its validation gate.
+    """
+    raw = _unb64(public_key_b64)
+    Ed25519PublicKey.from_public_bytes(raw)  # validates; raises on garbage
+    digest = hashlib.sha256(raw).hexdigest()
+    return ":".join(digest[i : i + 4] for i in range(0, 16, 4))
+
+
+def verify_signature(public_key_b64: str, payload: bytes, signature_b64: str) -> bool:
+    """True iff ``signature`` is valid for ``payload`` under the given key."""
+    try:
+        key = Ed25519PublicKey.from_public_bytes(_unb64(public_key_b64))
+        key.verify(_unb64(signature_b64), payload)
+        return True
+    except Exception:
+        return False
+
+
+def verify_manifest(manifest: Dict[str, Any], signature_block: Dict[str, Any]) -> bool:
+    canonical = json.dumps(manifest, sort_keys=True, ensure_ascii=False).encode("utf-8")
+    return verify_signature(
+        str(signature_block.get("public_key") or ""),
+        canonical,
+        str(signature_block.get("signature") or ""),
+    )
+
+
+__all__ = ["DeviceIdentity", "verify_signature", "verify_manifest"]