ltcai 3.6.0 → 4.0.1

package/latticeai/brain/store.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+from .documents import KnowledgeGraphDocumentsMixin
+from .discovery import KnowledgeGraphDiscoveryMixin
+from .ingest import KnowledgeGraphIngestMixin
+from .projection import KnowledgeGraphProjectionMixin
+from .provenance import KnowledgeGraphProvenanceMixin
+from .retrieval import KnowledgeGraphRetrievalMixin
+from .write_master import KnowledgeGraphWriteMixin
+
+
+class KnowledgeGraphStore(
+    KnowledgeGraphProjectionMixin,
+    KnowledgeGraphWriteMixin,
+    KnowledgeGraphDiscoveryMixin,
+    KnowledgeGraphIngestMixin,
+    KnowledgeGraphProvenanceMixin,
+    KnowledgeGraphDocumentsMixin,
+    KnowledgeGraphRetrievalMixin,
+):
+    def __init__(self, db_path: Path, blob_dir: Path, embedder: Any = None):
+        self.db_path = Path(db_path)
+        self.blob_dir = Path(blob_dir)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.blob_dir.mkdir(parents=True, exist_ok=True)
+        # The embedder is swappable behind a fixed interface
+        # (model_id/dim/embed/encode/decode/similarity). Defaults to the
+        # deterministic, offline hash model so the store works with no config;
+        # server_app injects a provider-backed embedder from Config.
+        self._embedding_model = (
+            embedder if embedder is not None else LocalEmbeddingModel()
+        )
+        self._init_db()
+        # Read graph queries from the v2 projection (kgv2_* views) when available.
+        # Toggle off (e.g. in tests) to compare against the legacy tables.
+        self._read_from_v2 = KGStoreV2 is not None and _READ_FROM_V2_DEFAULT
+
+    def _read_tables(self) -> tuple:
+        """Return (nodes_table, edges_table) for read queries.
+
+        Same read code runs against the legacy tables or the v2 reconstruction
+        views, so the two paths are equivalent by construction.
+        """
+        if self._read_from_v2:
+            return ("kgv2_nodes", "kgv2_edges")
+        return ("nodes", "edges")
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA foreign_keys=ON")
+        return conn
+
+    def _init_db(self) -> None:
+        with self._connect() as conn:
+            db_format = int(conn.execute("PRAGMA user_version").fetchone()[0] or 0)
+            if db_format > _KG_DB_FORMAT_VERSION:
+                raise RuntimeError(
+                    f"Knowledge Graph DB format {db_format} is newer than this build "
+                    f"({_KG_DB_FORMAT_VERSION}); restore a pre-upgrade backup or upgrade Lattice AI."
+                )
+            conn.executescript(
+                """
+                    CREATE TABLE IF NOT EXISTS graph_meta (
+                      key TEXT PRIMARY KEY,
+                      value TEXT NOT NULL
+                    );
+                    CREATE TABLE IF NOT EXISTS nodes (
+                      id TEXT PRIMARY KEY,
+                      type TEXT NOT NULL,
+                      title TEXT NOT NULL,
+                      summary TEXT,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      raw_json TEXT NOT NULL CHECK (json_valid(raw_json)),
+                      created_at TEXT NOT NULL,
+                      updated_at TEXT NOT NULL
+                    );
+                    CREATE TABLE IF NOT EXISTS edges (
+                      id TEXT PRIMARY KEY,
+                      from_node TEXT NOT NULL,
+                      to_node TEXT NOT NULL,
+                      type TEXT NOT NULL,
+                      weight REAL NOT NULL DEFAULT 1.0,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      created_at TEXT NOT NULL,
+                      UNIQUE(from_node, to_node, type),
+                      FOREIGN KEY(from_node) REFERENCES nodes(id) ON DELETE CASCADE,
+                      FOREIGN KEY(to_node) REFERENCES nodes(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS chunks (
+                      id TEXT PRIMARY KEY,
+                      source_node TEXT NOT NULL,
+                      text TEXT NOT NULL,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      created_at TEXT NOT NULL,
+                      FOREIGN KEY(source_node) REFERENCES nodes(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS knowledge_sources (
+                      id TEXT PRIMARY KEY,
+                      root_path TEXT NOT NULL UNIQUE,
+                      os_type TEXT NOT NULL,
+                      drive_id TEXT,
+                      label TEXT,
+                      status TEXT NOT NULL,
+                      include_ocr INTEGER NOT NULL DEFAULT 0,
+                      watch_enabled INTEGER NOT NULL DEFAULT 0,
+                      consent_json TEXT NOT NULL CHECK (json_valid(consent_json)),
+                      created_at TEXT NOT NULL,
+                      updated_at TEXT NOT NULL,
+                      last_scanned_at TEXT
+                    );
+                    CREATE TABLE IF NOT EXISTS local_file_index (
+                      id TEXT PRIMARY KEY,
+                      source_id TEXT NOT NULL,
+                      os_type TEXT NOT NULL,
+                      drive_id TEXT,
+                      root_path TEXT NOT NULL,
+                      file_path TEXT NOT NULL,
+                      relative_path TEXT NOT NULL,
+                      file_name TEXT NOT NULL,
+                      extension TEXT NOT NULL,
+                      size_bytes INTEGER,
+                      modified_at TEXT,
+                      sha256 TEXT,
+                      last_scanned_at TEXT,
+                      last_indexed_at TEXT,
+                      parser_type TEXT,
+                      status TEXT NOT NULL,
+                      error_message TEXT,
+                      graph_node_id TEXT,
+                      deleted INTEGER NOT NULL DEFAULT 0,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      UNIQUE(source_id, relative_path),
+                      FOREIGN KEY(source_id) REFERENCES knowledge_sources(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS vector_embeddings (
+                      item_id TEXT PRIMARY KEY,
+                      item_type TEXT NOT NULL,
+                      source_node TEXT NOT NULL,
+                      text_hash TEXT NOT NULL,
+                      embedding BLOB NOT NULL,
+                      embedding_dim INTEGER NOT NULL,
+                      embedding_model TEXT NOT NULL,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      indexed_at TEXT NOT NULL,
+                      FOREIGN KEY(source_node) REFERENCES nodes(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS vector_index_operations (
+                      id TEXT PRIMARY KEY,
+                      operation TEXT NOT NULL,
+                      status TEXT NOT NULL,
+                      requested_at TEXT NOT NULL,
+                      started_at TEXT,
+                      completed_at TEXT,
+                      items_total INTEGER NOT NULL DEFAULT 0,
+                      items_indexed INTEGER NOT NULL DEFAULT 0,
+                      items_skipped INTEGER NOT NULL DEFAULT 0,
+                      error_message TEXT,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json))
+                    );
+                    -- v3.6.0 Knowledge Graph First: per-ingestion provenance trail.
+                    -- Append-only audit of where every graph node came from, when it
+                    -- was captured, how it was processed, and whether it was embedded /
+                    -- linked / used by an agent. get_provenance() returns the latest row.
+                    CREATE TABLE IF NOT EXISTS ingestion_provenance (
+                      id TEXT PRIMARY KEY,
+                      node_id TEXT NOT NULL,
+                      source_type TEXT NOT NULL,
+                      source_uri TEXT,
+                      content_hash TEXT,
+                      title TEXT,
+                      pipeline TEXT NOT NULL,
+                      owner TEXT,
+                      workspace_id TEXT,
+                      captured_at TEXT,
+                      modified_at TEXT,
+                      embedded INTEGER NOT NULL DEFAULT 0,
+                      linked INTEGER NOT NULL DEFAULT 0,
+                      duplicate INTEGER NOT NULL DEFAULT 0,
+                      agent_used TEXT,
+                      chunk_count INTEGER NOT NULL DEFAULT 0,
+                      permissions_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(permissions_json)),
+                      metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)),
+                      created_at TEXT NOT NULL
+                    );
+                    CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
+                    CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node);
+                    CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node);
+                    CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_node);
+                    CREATE INDEX IF NOT EXISTS idx_knowledge_sources_root ON knowledge_sources(root_path);
+                    CREATE INDEX IF NOT EXISTS idx_local_file_index_source ON local_file_index(source_id);
+                    CREATE INDEX IF NOT EXISTS idx_local_file_index_status ON local_file_index(status);
+                    CREATE INDEX IF NOT EXISTS idx_local_file_index_graph_node ON local_file_index(graph_node_id);
+                    CREATE INDEX IF NOT EXISTS idx_vector_embeddings_type ON vector_embeddings(item_type);
+                    CREATE INDEX IF NOT EXISTS idx_vector_embeddings_source ON vector_embeddings(source_node);
+                    CREATE INDEX IF NOT EXISTS idx_vector_embeddings_model ON vector_embeddings(embedding_model);
+                    CREATE INDEX IF NOT EXISTS idx_vector_index_operations_requested ON vector_index_operations(requested_at);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_node ON ingestion_provenance(node_id);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_source_type ON ingestion_provenance(source_type);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_hash ON ingestion_provenance(content_hash);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_created ON ingestion_provenance(created_at);
+                    """
+            )
+            conn.execute(
+                "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+                ("schema_version", str(GRAPH_SCHEMA_VERSION)),
+            )
+        self._init_v2_schema()
+        self._init_fts()
+
+
+__all__ = ["KnowledgeGraphStore"]

package/latticeai/brain/write_master.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphWriteMixin:
+    def _upsert_node(
+        self,
+        conn: sqlite3.Connection,
+        node_id: str,
+        node_type: str,
+        title: str,
+        summary: str = "",
+        metadata: Optional[Dict[str, Any]] = None,
+        raw: Optional[Dict[str, Any]] = None,
+        owner: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        visibility: Optional[str] = None,
+    ) -> str:
+        now = _now()
+        # v4 write-mastering: nodes_v2 is authoritative; the legacy nodes
+        # table is maintained as the compatibility projection.
+        title_s = title[:240]
+        summary_s = summary[:1000]
+        meta_json = _json(metadata)
+        self._v2_project_node(
+            conn,
+            node_id,
+            node_type,
+            title_s,
+            summary_s,
+            meta_json,
+            created_at=now,
+            updated_at=now,
+            owner=owner,
+            workspace_id=workspace_id,
+            visibility=visibility,
+            strict=True,
+        )
+        conn.execute(
+            """
+                INSERT INTO nodes(id, type, title, summary, metadata_json, raw_json, created_at, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(id) DO UPDATE SET
+                  title=excluded.title,
+                  summary=excluded.summary,
+                  metadata_json=excluded.metadata_json,
+                  raw_json=excluded.raw_json,
+                  updated_at=excluded.updated_at
+                """,
+            (node_id, node_type, title_s, summary_s, meta_json, _json(raw), now, now),
+        )
+        if node_type != "Chunk":
+            self._upsert_vector_item(
+                conn,
+                item_id=node_id,
+                item_type="node",
+                source_node=node_id,
+                text=self._vector_text_for_node(
+                    title=title_s, summary=summary_s, metadata=metadata
+                ),
+                metadata={"node_type": node_type, **(metadata or {})},
+            )
+        return node_id
+
+    def _upsert_edge(
+        self,
+        conn: sqlite3.Connection,
+        from_node: str,
+        to_node: str,
+        edge_type: str,
+        weight: float = 1.0,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        # v4 write door: every new edge stores the canonical EdgeType value —
+        # free-string types (e.g. '포함함', '언급함') are normalized here, so no
+        # caller can mint new legacy taxonomy. The original label survives in
+        # metadata.legacy_label for traceability.
+        if EdgeType is not None:
+            canonical = EdgeType.from_legacy(edge_type).value
+            if canonical != edge_type:
+                metadata = dict(metadata or {})
+                metadata.setdefault("legacy_label", edge_type)
+            edge_type = canonical
+        edge_id = f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}')[:24]}"
+        now = _now()
+        meta_json = _json(metadata)  # canonical string shared with the projection
+        self._v2_project_edge(
+            conn,
+            from_node,
+            to_node,
+            edge_type,
+            float(weight),
+            meta_json,
+            edge_id=edge_id,
+            created_at=now,
+            strict=True,
+        )
+        conn.execute(
+            """
+                INSERT INTO edges(id, from_node, to_node, type, weight, metadata_json, created_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(from_node, to_node, type) DO UPDATE SET
+                  weight=max(edges.weight, excluded.weight),
+                  metadata_json=excluded.metadata_json
+                """,
+            (edge_id, from_node, to_node, edge_type, float(weight), meta_json, now),
+        )
+        return edge_id
+
+    def _vector_text_for_node(
+        self,
+        *,
+        title: str,
+        summary: str = "",
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        metadata = metadata or {}
+        meta_parts = []
+        for key in (
+            "filename",
+            "relative_path",
+            "file_path",
+            "conversation_id",
+            "source",
+            "category",
+            "ext",
+            "role",
+        ):
+            value = metadata.get(key)
+            if value:
+                meta_parts.append(str(value))
+        return _clean_text(
+            "\n".join([str(title or ""), str(summary or ""), " ".join(meta_parts)])
+        )
+
+    def _upsert_vector_item(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        item_id: str,
+        item_type: str,
+        source_node: str,
+        text: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> bool:
+        text = _clean_text(text)
+        if len(text) < 2:
+            conn.execute("DELETE FROM vector_embeddings WHERE item_id=?", (item_id,))
+            return False
+        text_hash = _sha256_text(text)
+        existing = conn.execute(
+            """
+                SELECT text_hash, embedding_dim, embedding_model
+                FROM vector_embeddings
+                WHERE item_id=?
+                """,
+            (item_id,),
+        ).fetchone()
+        if (
+            existing
+            and existing["text_hash"] == text_hash
+            and existing["embedding_dim"] == self._embedding_model.dim
+            and existing["embedding_model"] == self._embedding_model.model_id
+        ):
+            return False
+        embedding = self._embedding_model.encode(
+            self._embedding_model.embed(text[:50_000])
+        )
+        conn.execute(
+            """
+                INSERT INTO vector_embeddings(
+                  item_id, item_type, source_node, text_hash, embedding,
+                  embedding_dim, embedding_model, metadata_json, indexed_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(item_id) DO UPDATE SET
+                  item_type=excluded.item_type,
+                  source_node=excluded.source_node,
+                  text_hash=excluded.text_hash,
+                  embedding=excluded.embedding,
+                  embedding_dim=excluded.embedding_dim,
+                  embedding_model=excluded.embedding_model,
+                  metadata_json=excluded.metadata_json,
+                  indexed_at=excluded.indexed_at
+                """,
+            (
+                item_id,
+                item_type,
+                source_node,
+                text_hash,
+                embedding,
+                self._embedding_model.dim,
+                self._embedding_model.model_id,
+                _json(metadata),
+                _now(),
+            ),
+        )
+        return True
+
+    def _upsert_chunk(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        chunk_id: str,
+        source_node: str,
+        text: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        metadata = metadata or {}
+        conn.execute(
+            "INSERT OR REPLACE INTO chunks(id, source_node, text, metadata_json, created_at) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (chunk_id, source_node, text, _json(metadata), _now()),
+        )
+        self._upsert_vector_item(
+            conn,
+            item_id=chunk_id,
+            item_type="chunk",
+            source_node=chunk_id,
+            text=text,
+            metadata={**metadata, "parent_source_node": source_node},
+        )

package/latticeai/core/agent.py

@@ -21,6 +21,7 @@ only owns the state machine.
 from __future__ import annotations
 
 import json
+import logging
 import re
 import subprocess
 from dataclasses import dataclass
@@ -128,6 +129,12 @@ class AgentDeps:
     # lifecycle, so the agent tool path no longer bypasses hooks.
     hooks: Any = None
 
+    # ── brain memory port (optional) ─────────────────────────────────
+    # When present, completed-run learnings become typed Experience records
+    # through the unified ingestion pipeline (with provenance), replacing
+    # the vault markdown dump.
+    brain_memory: Any = None
+
 
 class AgentRuntime:
     """Drives the agent state machine over injected :class:`AgentDeps`."""
@@ -429,13 +436,30 @@ class AgentRuntime:
             )
             mem = extract_action(str(raw))
             if mem.get("save_to_knowledge") and mem.get("learnings"):
-                d.knowledge_save(
-                    "\n".join(mem["learnings"]),
-                    folder="30_Projects",
-                    title=f"Agent: {req.message[:60]}",
-                )
-        except Exception:
-            pass
+                learnings = "\n".join(mem["learnings"])
+                if d.brain_memory is not None:
+                    # This runtime is LLM-driven — its learnings are real
+                    # experiences and enter the brain with provenance.
+                    d.brain_memory.record_experience(
+                        f"Agent: {req.message[:60]}",
+                        learnings,
+                        run={
+                            "mode": "llm",
+                            "status": "ok",
+                            "agent_id": "agent:executor",
+                            "steps": len(ctx.transcript),
+                        },
+                        user_email=current_user or None,
+                    )
+                else:
+                    d.knowledge_save(
+                        learnings,
+                        folder="30_Projects",
+                        title=f"Agent: {req.message[:60]}",
+                    )
+        except Exception as exc:
+            # Never crash a completed run, but never swallow silently either.
+            logging.warning("agent memory update failed: %s", exc)
 
     # ── DRIVE LOOP ───────────────────────────────────────────────────
     async def run_to_completion(

package/latticeai/core/audit.py

@@ -5,7 +5,6 @@ import logging
 import os
 import re
 import threading
-from datetime import datetime
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional
 
@@ -217,12 +216,6 @@ def build_admin_audit_report(
                 u["high_sensitive_events"] += 1
             sensitive_events.append(_public_audit_event(event))
 
-    allowed_keys = {
-        "event_type", "timestamp", "role", "user_email", "user_nickname", "source",
-        "conversation_id", "command", "scope", "target_email", "filename", "mime_type",
-        "ext", "bytes", "extracted_chars", "graph_node", "keep_last", "removed", "kept",
-        "started_at", "sensitivity", "sensitive_labels", "content_preview", "content_chars",
-    }
     recent = [_public_audit_event(e) for e in events[-50:]]
 
     result: Dict[str, Any] = {

package/latticeai/core/config.py

@@ -19,7 +19,7 @@ concept and are read dynamically at call time.
 from __future__ import annotations
 
 import sys
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Mapping, Optional
 

package/latticeai/core/context_builder.py

@@ -7,9 +7,8 @@ retrieve_context_for_generation() 파이프라인:
   Step 3: Top-K 결과를 구조화된 Markdown Context로 변환
 """
 
-import logging
 import re
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 _CLEAN_RE = re.compile(r"\s+")
 

package/latticeai/core/enterprise.py

@@ -22,7 +22,7 @@ from __future__ import annotations
 
 import os
 from enum import Enum
-from typing import Dict, List, Optional, Protocol, runtime_checkable
+from typing import Dict, List, Protocol, runtime_checkable
 
 
 class Edition(str, Enum):

package/latticeai/core/graph_curator.py

@@ -20,8 +20,8 @@ import logging
 import math
 import re
 import time
-from dataclasses import dataclass, field, asdict
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Set
 
 logger = logging.getLogger(__name__)