ltcai 0.5.0 → 0.6.0

package/knowledge_graph.py

@@ -23,14 +23,23 @@ from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 try:
-    from kg_schema import KGStoreV2
+    from kg_schema import KGStoreV2, NodeType, EdgeType, _exec_script
 except Exception:  # pragma: no cover - v2 schema is optional at import time
     KGStoreV2 = None  # type: ignore[assignment]
+    NodeType = None   # type: ignore[assignment]
+    EdgeType = None   # type: ignore[assignment]
+    _exec_script = None  # type: ignore[assignment]
 
 # Default read source for the graph queries: v2 reconstruction views.
 # Override with LATTICEAI_KG_READ_V2=0 to fall back to the legacy tables.
 _READ_FROM_V2_DEFAULT = os.getenv("LATTICEAI_KG_READ_V2", "1") != "0"
 
+# Bump when the v2 projection layout changes (columns, normalization rules).
+# On init, a stale projection is dropped and rebuilt from the authoritative
+# legacy tables — safe because nodes_v2/edges_v2 only ever hold a derived view.
+# v4: summary nullable + verbatim (byte-faithful) projection of legacy values.
+_PROJECTION_VERSION = 4
+
 _llm_router_ref = None
 
 def set_llm_router(router_instance):
@@ -916,133 +925,192 @@ class KnowledgeGraphStore:
             )
         self._init_v2_schema()
 
-    # SQL views that reconstruct the *exact* legacy row shape on top of the v2
-    # tables, so the read methods can run unchanged against either source. The
-    # projection (see _v2_project_node/_edge) stashes summary + the original
-    # metadata_json + (via the type column) the legacy type string, so these
-    # views are byte-faithful to the legacy nodes/edges tables.
+    # SQL views that reconstruct the *exact* legacy row shape on top of the
+    # normalized v2 tables, so the read methods run unchanged against either
+    # source. The projection stores the raw legacy type string in ``legacy_type``
+    # and promotes summary + metadata to first-class columns (no more
+    # ``attrs._kg`` passthrough / ``evidence`` abuse), so these views are
+    # byte-faithful to the legacy nodes/edges tables.
     _V2_VIEWS_SQL = """
     CREATE VIEW IF NOT EXISTS kgv2_nodes AS
-      SELECT id, type,
+      SELECT id,
+             COALESCE(legacy_type, type) AS type,
              label AS title,
-             COALESCE(json_extract(attrs, '$._kg.summary'), '')   AS summary,
-             COALESCE(json_extract(attrs, '$._kg.metadata_json'), '{}') AS metadata_json,
+             summary,
+             attrs AS metadata_json,
              created_at, updated_at
       FROM nodes_v2;
     CREATE VIEW IF NOT EXISTS kgv2_edges AS
-      SELECT id, source AS from_node, target AS to_node, type, weight,
-             COALESCE(evidence, '{}') AS metadata_json, created_at
+      SELECT id, source AS from_node, target AS to_node,
+             COALESCE(legacy_type, type) AS type,
+             weight,
+             metadata AS metadata_json,
+             created_at
       FROM edges_v2;
     """
 
     def _init_v2_schema(self) -> None:
-        """Initialize the v2 tables + reconstruction views and backfill from legacy.
-
-        Completes the v2 migration: both write (dual-write projection in
-        _upsert_node/_upsert_edge) and read (read methods route to the kgv2_*
-        views when ``_READ_FROM_V2`` is on) flow through the v2 tables. Legacy
-        nodes/edges are retained as the durable source until the v2 path bakes in.
+        """Initialize the normalized v2 tables + reconstruction views, migrating
+        the projection layout when it is stale — **atomically**.
+
+        The entire DROP → CREATE → VIEWS → BACKFILL → version-stamp sequence runs
+        in a single transaction on one connection: on any failure it rolls back,
+        leaving the prior projection untouched and the version unchanged, so the
+        next startup simply retries. The migration only ever touches the v2
+        tables/views and the ``projection_version`` key — never the authoritative
+        legacy ``nodes``/``edges`` — so legacy data cannot be corrupted even if
+        the rebuild fails midway.
         """
-        if KGStoreV2 is None:
+        if KGStoreV2 is None or _exec_script is None:
             return
         try:
-            KGStoreV2(self.db_path).init_schema()
             with self._connect() as conn:
-                conn.executescript(self._V2_VIEWS_SQL)
-            self._backfill_v2_if_needed()
+                conn.execute("BEGIN")
+                stale = self._projection_version(conn) != _PROJECTION_VERSION
+                if stale:
+                    # The projection is non-authoritative; drop it so init_schema
+                    # recreates the tables with the current normalized columns.
+                    for stmt in (
+                        "DROP VIEW IF EXISTS kgv2_edges",
+                        "DROP VIEW IF EXISTS kgv2_nodes",
+                        "DROP TABLE IF EXISTS edges_v2",
+                        "DROP TABLE IF EXISTS nodes_v2",
+                    ):
+                        conn.execute(stmt)
+                # init_schema(conn=...) joins this transaction (no implicit commit)
+                KGStoreV2(self.db_path).init_schema(conn=conn)
+                _exec_script(conn, self._V2_VIEWS_SQL)
+                self._backfill_v2_on(conn, force=stale)
+                # version stamp commits together with the backfill — never stranded
+                conn.execute(
+                    "INSERT OR REPLACE INTO kg_meta(key, value) VALUES ('projection_version', ?)",
+                    (str(_PROJECTION_VERSION),),
+                )
         except Exception as e:
             logging.warning("knowledge_graph: v2 schema init/backfill skipped: %s", e)
 
-    def _backfill_v2_if_needed(self) -> None:
-        """Project legacy nodes/edges into the v2 tables when v2 is empty or stale.
+    def _projection_version(self, conn: sqlite3.Connection) -> int:
+        """Return the stored v2 projection layout version (0 if unknown).
+
+        A fresh DB (kg_meta absent) raises ``sqlite3.OperationalError`` here and
+        is correctly treated as version 0 → rebuild. Only sqlite errors are
+        swallowed so a real bug doesn't masquerade as a stale projection.
+        """
+        try:
+            row = conn.execute(
+                "SELECT value FROM kg_meta WHERE key='projection_version'"
+            ).fetchone()
+            return int(row["value"]) if row and row["value"] is not None else 0
+        except sqlite3.Error:
+            return 0
+
+    def _backfill_v2_if_needed(self, *, force: bool = False) -> None:
+        """Project legacy nodes/edges into v2 on a fresh transaction.
 
-        Non-destructive to legacy. Reprojects when the v2 rows predate the
-        ``_kg`` reconstruction blob (older enum-only backfill), so the views
-        stay faithful. Idempotent: no-ops once v2 carries the current projection.
+        Thin wrapper around :meth:`_backfill_v2_on` for callers (tests, ad-hoc
+        re-sync) that aren't already inside the migration transaction.
         """
         try:
             with self._connect() as conn:
-                v2_nodes = conn.execute("SELECT COUNT(*) FROM nodes_v2").fetchone()[0]
-                legacy_nodes = conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
-                if legacy_nodes == 0:
-                    return
-                if v2_nodes > 0:
-                    has_kg = conn.execute(
-                        "SELECT COUNT(*) FROM nodes_v2 WHERE json_extract(attrs,'$._kg') IS NOT NULL"
-                    ).fetchone()[0]
-                    if has_kg > 0:
-                        return  # current projection already present
-                # (re)project: clear v2 graph (not authoritative) and rebuild
-                conn.execute("DELETE FROM edges_v2")
-                conn.execute("DELETE FROM nodes_v2")
-                n = e = 0
-                for r in conn.execute(
-                    "SELECT id, type, title, summary, metadata_json, created_at, updated_at FROM nodes"
-                ).fetchall():
-                    self._v2_project_node(
-                        conn, r["id"], r["type"], r["title"] or "", r["summary"] or "",
-                        _safe_loads(r["metadata_json"]),
-                        created_at=r["created_at"], updated_at=r["updated_at"],
-                    )
-                    n += 1
-                for r in conn.execute(
-                    "SELECT id, from_node, to_node, type, weight, metadata_json, created_at FROM edges"
-                ).fetchall():
-                    self._v2_project_edge(
-                        conn, r["from_node"], r["to_node"], r["type"], float(r["weight"] or 1.0),
-                        _safe_loads(r["metadata_json"]), edge_id=r["id"], created_at=r["created_at"],
-                    )
-                    e += 1
-                logging.info("knowledge_graph: projected legacy → v2 (%d nodes, %d edges)", n, e)
+                self._backfill_v2_on(conn, force=force)
         except Exception as ex:
             logging.warning("knowledge_graph: v2 backfill skipped: %s", ex)
 
-    # ── v2 dual-write projection (legacy types + summary/metadata in attrs._kg) ──
+    def _backfill_v2_on(self, conn: sqlite3.Connection, *, force: bool = False) -> None:
+        """Project legacy nodes/edges into the normalized v2 tables on ``conn``.
+
+        Non-destructive to legacy. ``force`` rebuilds unconditionally (used after
+        a layout migration); otherwise it only projects when v2 is empty. The v2
+        graph is a derived projection, so clearing + rebuilding it is always safe.
+        Idempotent: no-ops once v2 carries the current projection. Copies the
+        legacy column values **verbatim** so the kgv2_* views are byte-faithful.
+        """
+        legacy_nodes = conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
+        if legacy_nodes == 0:
+            return
+        v2_nodes = conn.execute("SELECT COUNT(*) FROM nodes_v2").fetchone()[0]
+        if v2_nodes > 0 and not force:
+            return  # current projection already present
+        # (re)project: clear v2 graph (not authoritative) and rebuild
+        conn.execute("DELETE FROM edges_v2")
+        conn.execute("DELETE FROM nodes_v2")
+        n = e = 0
+        for r in conn.execute(
+            "SELECT id, type, title, summary, metadata_json, created_at, updated_at FROM nodes"
+        ).fetchall():
+            self._v2_project_node(
+                conn, r["id"], r["type"], r["title"], r["summary"], r["metadata_json"],
+                created_at=r["created_at"], updated_at=r["updated_at"],
+            )
+            n += 1
+        for r in conn.execute(
+            "SELECT id, from_node, to_node, type, weight, metadata_json, created_at FROM edges"
+        ).fetchall():
+            self._v2_project_edge(
+                conn, r["from_node"], r["to_node"], r["type"], float(r["weight"] or 1.0),
+                r["metadata_json"], edge_id=r["id"], created_at=r["created_at"],
+            )
+            e += 1
+        logging.info("knowledge_graph: projected legacy → v2 (%d nodes, %d edges)", n, e)
+
+    # ── v2 dual-write projection (normalized type, byte-faithful legacy values) ──
+    # The projection stores the legacy ``title``/``summary``/``metadata_json``
+    # values it is handed VERBATIM (no truncation or JSON re-encoding) so the
+    # kgv2_* views reproduce the legacy rows exactly. Callers (_upsert_* and the
+    # backfill) pass the already-canonical legacy column values.
     def _v2_project_node(
         self, conn: sqlite3.Connection, node_id: str, node_type: str, title: str,
-        summary: str, metadata: Optional[Dict[str, Any]],
+        summary: Optional[str], metadata_json: Optional[str],
         *, created_at: Optional[str] = None, updated_at: Optional[str] = None,
     ) -> None:
         if KGStoreV2 is None:
             return
         ts = updated_at or _now()
-        attrs = _json({"_kg": {"summary": (summary or "")[:1000], "metadata_json": _json(metadata)}})
+        norm_type = NodeType.from_legacy(node_type).value if NodeType is not None else node_type
         try:
             conn.execute(
                 """
-                INSERT INTO nodes_v2(id, type, label, attrs, owner_id, visibility,
-                                     created_at, updated_at, importance_score)
-                VALUES (?, ?, ?, ?, NULL, 'private', ?, ?, 0.0)
+                INSERT INTO nodes_v2(id, type, legacy_type, label, summary, attrs,
+                                     owner_id, visibility, created_at, updated_at,
+                                     importance_score)
+                VALUES (?, ?, ?, ?, ?, ?, NULL, 'private', ?, ?, 0.0)
                 ON CONFLICT(id) DO UPDATE SET
-                  type=excluded.type, label=excluded.label,
+                  type=excluded.type, legacy_type=excluded.legacy_type,
+                  label=excluded.label, summary=excluded.summary,
                   attrs=excluded.attrs, updated_at=excluded.updated_at
                 """,
-                (node_id, node_type, (title or "")[:240], attrs, created_at or ts, ts),
+                (node_id, norm_type, node_type, title, summary,
+                 metadata_json if metadata_json is not None else "{}",
+                 created_at or ts, ts),
             )
         except Exception as ex:
             logging.debug("knowledge_graph: v2 node projection skipped (%s): %s", node_id, ex)
 
     def _v2_project_edge(
         self, conn: sqlite3.Connection, from_node: str, to_node: str, edge_type: str,
-        weight: float, metadata: Optional[Dict[str, Any]],
+        weight: float, metadata_json: Optional[str],
         *, edge_id: Optional[str] = None, created_at: Optional[str] = None,
     ) -> None:
         if KGStoreV2 is None:
             return
-        meta = metadata or {}
         eid = edge_id or f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}')[:24]}"
+        norm_type = EdgeType.from_legacy(edge_type).value if EdgeType is not None else edge_type
+        meta_str = metadata_json if metadata_json is not None else "{}"
+        confidence = float(_safe_loads(meta_str).get("confidence", 1.0))
         try:
             conn.execute(
                 """
-                INSERT INTO edges_v2(id, source, target, type, weight, confidence,
-                                     evidence, created_by, created_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, 'legacy', ?)
-                ON CONFLICT(source, target, type) DO UPDATE SET
+                INSERT INTO edges_v2(id, source, target, type, legacy_type, weight,
+                                     confidence, evidence, metadata, created_by, created_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, '[]', ?, 'legacy', ?)
+                ON CONFLICT(source, target, legacy_type) DO UPDATE SET
+                  type=excluded.type,
                   weight=max(edges_v2.weight, excluded.weight),
-                  evidence=excluded.evidence
+                  confidence=excluded.confidence,
+                  metadata=excluded.metadata
                 """,
-                (eid, from_node, to_node, edge_type, float(weight),
-                 float(meta.get("confidence", 1.0)), _json(meta), created_at or _now()),
+                (eid, from_node, to_node, norm_type, edge_type, float(weight),
+                 confidence, meta_str, created_at or _now()),
             )
         except Exception as ex:
             logging.debug("knowledge_graph: v2 edge projection skipped (%s->%s): %s", from_node, to_node, ex)
@@ -1069,6 +1137,35 @@ class KnowledgeGraphStore:
         except Exception as ex:
             logging.debug("knowledge_graph: v2 edge delete mirror skipped: %s", ex)
 
+    def _v2_sync_report(self) -> Dict[str, Any]:
+        """Diagnose the dual-write invariant: legacy node/edge id sets must equal
+        the v2 projection's. Returns counts + any drift (ids missing from / extra
+        in v2). ``in_sync`` is True only when both id sets match exactly.
+
+        All legacy writes go through _upsert_node/_upsert_edge (which dual-write)
+        and every legacy delete is mirrored, so a non-empty drift signals a
+        bypassed write path — this is the runtime guard for that invariant.
+        """
+        if KGStoreV2 is None:
+            return {"available": False, "in_sync": True}
+        with self._connect() as conn:
+            legacy_nodes = {r[0] for r in conn.execute("SELECT id FROM nodes")}
+            v2_nodes = {r[0] for r in conn.execute("SELECT id FROM nodes_v2")}
+            legacy_edges = {r[0] for r in conn.execute("SELECT id FROM edges")}
+            v2_edges = {r[0] for r in conn.execute("SELECT id FROM edges_v2")}
+        return {
+            "available": True,
+            "in_sync": legacy_nodes == v2_nodes and legacy_edges == v2_edges,
+            "nodes_legacy": len(legacy_nodes),
+            "nodes_v2": len(v2_nodes),
+            "edges_legacy": len(legacy_edges),
+            "edges_v2": len(v2_edges),
+            "nodes_missing_from_v2": sorted(legacy_nodes - v2_nodes),
+            "nodes_extra_in_v2": sorted(v2_nodes - legacy_nodes),
+            "edges_missing_from_v2": sorted(legacy_edges - v2_edges),
+            "edges_extra_in_v2": sorted(v2_edges - legacy_edges),
+        }
+
     def _upsert_node(
         self,
         conn: sqlite3.Connection,
@@ -1080,6 +1177,11 @@ class KnowledgeGraphStore:
         raw: Optional[Dict[str, Any]] = None,
     ) -> str:
         now = _now()
+        # Canonical stored values, computed once and shared with the v2
+        # projection so legacy and v2 hold byte-identical strings.
+        title_s = title[:240]
+        summary_s = summary[:1000]
+        meta_json = _json(metadata)
         conn.execute(
             """
             INSERT INTO nodes(id, type, title, summary, metadata_json, raw_json, created_at, updated_at)
@@ -1091,10 +1193,10 @@ class KnowledgeGraphStore:
               raw_json=excluded.raw_json,
               updated_at=excluded.updated_at
             """,
-            (node_id, node_type, title[:240], summary[:1000], _json(metadata), _json(raw), now, now),
+            (node_id, node_type, title_s, summary_s, meta_json, _json(raw), now, now),
         )
         # dual-write: project into the v2 graph on the same transaction
-        self._v2_project_node(conn, node_id, node_type, title, summary, metadata,
+        self._v2_project_node(conn, node_id, node_type, title_s, summary_s, meta_json,
                               created_at=now, updated_at=now)
         return node_id
 
@@ -1109,6 +1211,7 @@ class KnowledgeGraphStore:
     ) -> str:
         edge_id = f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}')[:24]}"
         now = _now()
+        meta_json = _json(metadata)   # canonical string shared with the projection
         conn.execute(
             """
             INSERT INTO edges(id, from_node, to_node, type, weight, metadata_json, created_at)
@@ -1117,10 +1220,10 @@ class KnowledgeGraphStore:
               weight=max(edges.weight, excluded.weight),
               metadata_json=excluded.metadata_json
             """,
-            (edge_id, from_node, to_node, edge_type, float(weight), _json(metadata), now),
+            (edge_id, from_node, to_node, edge_type, float(weight), meta_json, now),
         )
         # dual-write: project into the v2 graph on the same transaction
-        self._v2_project_edge(conn, from_node, to_node, edge_type, float(weight), metadata,
+        self._v2_project_edge(conn, from_node, to_node, edge_type, float(weight), meta_json,
                               edge_id=edge_id, created_at=now)
         return edge_id
 
@@ -3072,7 +3175,7 @@ class KnowledgeGraphStore:
                 conn.execute(
                     """
                     DELETE FROM nodes_v2
-                    WHERE type='Topic'
+                    WHERE legacy_type='Topic'
                       AND id NOT IN (SELECT target FROM edges_v2)
                       AND id NOT IN (SELECT source FROM edges_v2)
                     """

package/latticeai/core/agent.py

@@ -8,13 +8,13 @@ no globals, and no I/O of its own — every collaborator is injected through
 
 Two adapters justify the seam:
 
-* production wires ``AgentDeps`` from server.py's ``LLMRouter``, governance
+* production wires ``AgentDeps`` from ``latticeai.server_app``'s ``LLMRouter``, governance
   map, audit log, and prompts;
 * tests pass fake ports (an LLM that returns canned JSON, a recording tool
   executor) and drive a full PLAN→EXECUTE→VERIFY→DONE cycle without a server.
 
 HTTP concerns — request parsing, chat-history persistence, response shaping,
-scheduling the background memory update — stay in server.py. This module
+scheduling the background memory update — stay in the app layer. This module
 only owns the state machine.
 """
 

package/latticeai/core/agent_prompts.py

@@ -0,0 +1,101 @@
+"""Role prompts for the Lattice multi-role agent runtime."""
+
+from __future__ import annotations
+
+from latticeai.core.tool_registry import TOOL_CATALOG_BRIEF
+
+
+PLANNER_PROMPT = """You are the PLANNER role in Lattice AI's multi-role agent harness.
+Your ONLY job: analyze the request and produce a structured execution plan.
+You do NOT call tools or write code.
+
+Respond with exactly ONE JSON object (no markdown, no fences):
+{
+  "action": "plan",
+  "state": "PLANNING",
+  "goal": "one-sentence goal in the user's language",
+  "steps": [
+    {"id": 1, "description": "what this step does", "action": "expected_tool", "purpose": "why needed"}
+  ],
+  "requires_approval": true,
+  "rollback_strategy": "git",
+  "estimated_steps": 3
+}
+
+Rules:
+- requires_approval = true if ANY step uses write/exec tools (edit_file, write_file, run_command, etc.)
+- rollback_strategy = "git" if steps modify existing files; "none" otherwise
+- Keep steps realistic: 2-4 for simple tasks, up to 10 for complex ones
+- Do NOT specify full tool args -- that is the Executor's job
+
+Available tools:""" + TOOL_CATALOG_BRIEF
+
+
+EXECUTOR_PROMPT = """You are the EXECUTOR role in Lattice AI's multi-role agent harness.
+You have a plan from the Planner. Execute it step by step using exactly one tool per response.
+
+You think and act like a senior software engineer:
+- Read (read_file, grep) BEFORE editing -- never guess at file contents
+- Prefer edit_file over write_file for existing files
+- Keep changes small and precise
+- Verify after changes with build_project or run_command
+
+Respond with exactly ONE JSON object per step:
+{"thoughts": "what you learned / why this next action", "action": "tool_name", "args": {...}}
+
+When the task is fully done AND a tool result in this run confirms it:
+{"thoughts": "verified", "action": "final", "message": "한국어로 무엇을 했고 어디서 검증했는지 요약"}
+
+ANTI-PATTERNS (will halt the loop):
+- Editing without reading first -> read_file + grep BEFORE edit_file
+- Repeating the same action+args -> check the transcript
+- Claiming done without a verification tool result in transcript
+- Hallucinating imports or file paths that were never confirmed by a tool result
+
+Available tools:""" + TOOL_CATALOG_BRIEF
+
+
+CRITIC_PROMPT = """You are the CRITIC / REVIEWER role in Lattice AI's multi-role agent harness.
+Review the execution transcript and determine whether the goal was achieved.
+
+Respond with exactly ONE JSON object:
+{
+  "action": "verdict",
+  "state": "VERIFYING",
+  "verdict": "PASS",
+  "reason": "why you think it passed or failed (cite specific tool results)",
+  "corrections": [],
+  "confidence": 0.95,
+  "next_state": "DONE"
+}
+
+verdict: "PASS" | "FAIL"
+next_state:
+  "DONE"      -- task succeeded; finish
+  "EXECUTING" -- task failed but corrections can fix it (use corrections field for retry)
+  "ROLLBACK"  -- task failed AND file changes should be undone
+
+Criteria for PASS: a tool result in the transcript explicitly confirms success.
+Be strict. Claiming done without evidence = FAIL."""
+
+
+MEMORY_UPDATER_PROMPT = """You are the MEMORY UPDATER role in Lattice AI's multi-role agent harness.
+After a completed task, extract reusable learnings.
+
+Respond with exactly ONE JSON object:
+{
+  "action": "memory",
+  "state": "DONE",
+  "learnings": ["one concise fact about this codebase or task"],
+  "artifacts": ["relative/path/to/created_or_modified_file"],
+  "save_to_knowledge": false
+}
+
+Rules:
+- max 5 learnings, one sentence each
+- save_to_knowledge = true only if learnings are genuinely useful across future sessions
+- artifacts = files the Executor actually created or modified (from transcript)
+"""
+
+
+AGENT_SYSTEM_PROMPT = EXECUTOR_PROMPT