ltcai 3.6.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/README.md +11 -7
  2. package/docs/V4_BRAIN_ARCHITECTURE.md +322 -0
  3. package/docs/V4_DIGITAL_BRAIN_RECOVERY.md +509 -0
  4. package/docs/V4_IMPLEMENTATION_PLAN.md +470 -0
  5. package/docs/kg-schema.md +47 -53
  6. package/kg_schema.py +93 -10
  7. package/knowledge_graph.py +362 -33
  8. package/knowledge_graph_api.py +11 -127
  9. package/latticeai/__init__.py +1 -1
  10. package/latticeai/api/admin.py +1 -1
  11. package/latticeai/api/agents.py +7 -1
  12. package/latticeai/api/auth.py +27 -4
  13. package/latticeai/api/chat.py +112 -76
  14. package/latticeai/api/health.py +1 -1
  15. package/latticeai/api/hooks.py +1 -1
  16. package/latticeai/api/knowledge_graph.py +146 -0
  17. package/latticeai/api/local_files.py +1 -1
  18. package/latticeai/api/mcp.py +23 -11
  19. package/latticeai/api/memory.py +1 -1
  20. package/latticeai/api/models.py +1 -1
  21. package/latticeai/api/network.py +81 -0
  22. package/latticeai/api/realtime.py +1 -1
  23. package/latticeai/api/search.py +26 -2
  24. package/latticeai/api/security_dashboard.py +2 -3
  25. package/latticeai/api/setup.py +2 -2
  26. package/latticeai/api/static_routes.py +2 -4
  27. package/latticeai/api/tools.py +3 -0
  28. package/latticeai/api/workflow_designer.py +46 -0
  29. package/latticeai/api/workspace.py +71 -49
  30. package/latticeai/app_factory.py +1710 -0
  31. package/latticeai/brain/__init__.py +18 -0
  32. package/latticeai/brain/context.py +213 -0
  33. package/latticeai/brain/conversations.py +236 -0
  34. package/latticeai/brain/identity.py +175 -0
  35. package/latticeai/brain/memory.py +102 -0
  36. package/latticeai/brain/network.py +205 -0
  37. package/latticeai/core/agent.py +31 -7
  38. package/latticeai/core/audit.py +0 -7
  39. package/latticeai/core/config.py +1 -1
  40. package/latticeai/core/context_builder.py +1 -2
  41. package/latticeai/core/enterprise.py +1 -1
  42. package/latticeai/core/graph_curator.py +2 -2
  43. package/latticeai/core/marketplace.py +1 -1
  44. package/latticeai/core/mcp_registry.py +791 -0
  45. package/latticeai/core/model_compat.py +1 -1
  46. package/latticeai/core/model_resolution.py +0 -1
  47. package/latticeai/core/multi_agent.py +238 -4
  48. package/latticeai/core/security.py +1 -1
  49. package/latticeai/core/sessions.py +37 -7
  50. package/latticeai/core/workflow_engine.py +114 -2
  51. package/latticeai/core/workspace_os.py +58 -10
  52. package/latticeai/models/__init__.py +7 -0
  53. package/latticeai/models/router.py +779 -0
  54. package/latticeai/server_app.py +29 -1536
  55. package/latticeai/services/agent_runtime.py +1 -0
  56. package/latticeai/services/app_context.py +75 -14
  57. package/latticeai/services/ingestion.py +47 -0
  58. package/latticeai/services/kg_portability.py +33 -3
  59. package/latticeai/services/memory_service.py +39 -11
  60. package/latticeai/services/model_runtime.py +2 -5
  61. package/latticeai/services/platform_runtime.py +100 -23
  62. package/latticeai/services/search_service.py +17 -8
  63. package/latticeai/services/tool_dispatch.py +12 -2
  64. package/latticeai/services/triggers.py +241 -0
  65. package/latticeai/services/upload_service.py +37 -12
  66. package/latticeai/services/workspace_service.py +31 -0
  67. package/llm_router.py +29 -772
  68. package/ltcai_cli.py +1 -2
  69. package/mcp_registry.py +25 -788
  70. package/p_reinforce.py +124 -14
  71. package/package.json +9 -7
  72. package/scripts/bump_version.py +99 -0
  73. package/scripts/generate_diagrams.py +0 -1
  74. package/scripts/lint_v3.mjs +82 -18
  75. package/scripts/validate_release_artifacts.py +0 -1
  76. package/scripts/wheel_smoke.py +142 -0
  77. package/server.py +11 -7
  78. package/setup_wizard.py +1142 -0
  79. package/static/account.html +2 -4
  80. package/static/admin.html +3 -5
  81. package/static/chat.html +3 -6
  82. package/static/graph.html +2 -4
  83. package/static/sw.js +81 -52
  84. package/static/v3/asset-manifest.json +20 -19
  85. package/static/v3/css/{lattice.base.e4cdd05d.css → lattice.base.49deefb5.css} +1 -1
  86. package/static/v3/css/lattice.base.css +1 -1
  87. package/static/v3/css/{lattice.components.9b49d614.css → lattice.components.cde18231.css} +1 -1
  88. package/static/v3/css/lattice.components.css +1 -1
  89. package/static/v3/css/{lattice.shell.8fcc9d33.css → lattice.shell.29d36d85.css} +1 -1
  90. package/static/v3/css/lattice.shell.css +1 -1
  91. package/static/v3/css/{lattice.tokens.e7018963.css → lattice.tokens.304cbc40.css} +3 -0
  92. package/static/v3/css/lattice.tokens.css +3 -0
  93. package/static/v3/css/{lattice.views.22f69117.css → lattice.views.0a18b6c5.css} +2 -2
  94. package/static/v3/css/lattice.views.css +2 -2
  95. package/static/v3/index.html +3 -4
  96. package/static/v3/js/{app.c541f955.js → app.356e6452.js} +1 -1
  97. package/static/v3/js/core/{api.33d6320e.js → api.7a308b89.js} +1 -1
  98. package/static/v3/js/core/{routes.2ce3815a.js → routes.7222343d.js} +22 -22
  99. package/static/v3/js/core/routes.js +22 -22
  100. package/static/v3/js/core/{shell.8c163e0e.js → shell.a1657f20.js} +4 -4
  101. package/static/v3/js/core/shell.js +1 -1
  102. package/static/v3/js/core/{store.34ebd5e6.js → store.204a08b2.js} +1 -1
  103. package/static/v3/js/core/store.js +1 -1
  104. package/static/v3/js/views/graph-canvas.17c15d65.js +509 -0
  105. package/static/v3/js/views/graph-canvas.js +509 -0
  106. package/static/v3/js/views/{hybrid-search.b22b97e0.js → hybrid-search.2fb63ed9.js} +1 -2
  107. package/static/v3/js/views/hybrid-search.js +1 -2
  108. package/static/v3/js/views/{knowledge-graph.a96040a5.js → knowledge-graph.5e40cbeb.js} +33 -37
  109. package/static/v3/js/views/knowledge-graph.js +33 -37
  110. package/static/vendor/chart.umd.min.js +20 -0
  111. package/static/vendor/fonts/inter-latin-300-normal.woff2 +0 -0
  112. package/static/vendor/fonts/inter-latin-400-normal.woff2 +0 -0
  113. package/static/vendor/fonts/inter-latin-500-normal.woff2 +0 -0
  114. package/static/vendor/fonts/inter-latin-600-normal.woff2 +0 -0
  115. package/static/vendor/fonts/inter-latin-700-normal.woff2 +0 -0
  116. package/static/vendor/fonts/inter-latin-800-normal.woff2 +0 -0
  117. package/static/vendor/fonts/inter.css +44 -0
  118. package/static/vendor/icons/tabler-icons.min.css +4 -0
  119. package/static/vendor/icons/tabler-icons.woff2 +0 -0
  120. package/static/vendor/marked.min.js +69 -0
  121. package/static/workspace.html +2 -2
  122. package/telegram_bot.py +1 -2
  123. package/tools/commands.py +4 -2
  124. package/tools/computer.py +1 -1
  125. package/tools/documents.py +1 -3
  126. package/tools/filesystem.py +0 -4
  127. package/tools/knowledge.py +1 -3
  128. package/tools/network.py +1 -3
  129. package/codex_telegram_bot.py +0 -195
  130. package/docs/assets/v3.4.0/agent-run.png +0 -0
  131. package/docs/assets/v3.4.0/agents.png +0 -0
  132. package/docs/assets/v3.4.0/before/chat-before.png +0 -0
  133. package/docs/assets/v3.4.0/before/files-before.png +0 -0
  134. package/docs/assets/v3.4.0/chat.png +0 -0
  135. package/docs/assets/v3.4.0/connect-folder.png +0 -0
  136. package/docs/assets/v3.4.0/files.png +0 -0
  137. package/docs/assets/v3.4.0/home.png +0 -0
  138. package/docs/assets/v3.4.0/hooks-dispatch.png +0 -0
  139. package/docs/assets/v3.4.0/knowledge-graph.png +0 -0
  140. package/docs/assets/v3.4.0/local-agent.png +0 -0
  141. package/docs/assets/v3.4.0/memory.png +0 -0
  142. package/docs/assets/v3.4.0/settings.png +0 -0
  143. package/docs/assets/v3.4.0/vision-input.png +0 -0
  144. package/docs/assets/v3.4.0/workflows.png +0 -0
  145. package/docs/assets/v3.4.1/e2e_runtime_log.txt +0 -42
  146. package/docs/assets/v3.4.1/hooks-dispatch.png +0 -0
  147. package/docs/assets/v3.4.1/local-agent.png +0 -0
  148. package/docs/images/admin-dashboard.png +0 -0
  149. package/docs/images/architecture.png +0 -0
  150. package/docs/images/enterprise.png +0 -0
  151. package/docs/images/graph.png +0 -0
  152. package/docs/images/hero.gif +0 -0
  153. package/docs/images/knowledge-graph.png +0 -0
  154. package/docs/images/lattice-ai-demo.gif +0 -0
  155. package/docs/images/lattice-ai-hero.png +0 -0
  156. package/docs/images/logo.svg +0 -33
  157. package/docs/images/mobile-responsive.png +0 -0
  158. package/docs/images/model-recommendation.png +0 -0
  159. package/docs/images/onboarding.png +0 -0
  160. package/docs/images/organization.png +0 -0
  161. package/docs/images/pipeline.png +0 -0
  162. package/docs/images/screenshot-admin.png +0 -0
  163. package/docs/images/screenshot-chat.png +0 -0
  164. package/docs/images/screenshot-graph.png +0 -0
  165. package/docs/images/skills.png +0 -0
  166. package/docs/images/workspace-dark.png +0 -0
  167. package/docs/images/workspace-light.png +0 -0
  168. package/docs/images/workspace.png +0 -0
  169. package/requirements.txt +0 -16
@@ -990,6 +990,62 @@ class KnowledgeGraphStore:
990
990
  ("schema_version", str(GRAPH_SCHEMA_VERSION)),
991
991
  )
992
992
  self._init_v2_schema()
993
+ self._init_fts()
994
+
995
+ # ── FTS5 keyword index (v4) ──────────────────────────────────────────
996
+ # Replaces LIKE '%q%' table scans for keyword search. The trigram
997
+ # tokenizer is required (not just FTS5): unicode61 indexes whole tokens
998
+ # and would silently break Korean substring recall ('프로젝트' must match
999
+ # '프로젝트를'). Without trigram support the store honestly reports
1000
+ # fts_enabled=False and the LIKE path remains authoritative.
1001
+ _FTS_SQL = """
1002
+ CREATE VIRTUAL TABLE IF NOT EXISTS node_fts USING fts5(
1003
+ node_id UNINDEXED, title, summary, metadata, tokenize='trigram'
1004
+ );
1005
+ CREATE TRIGGER IF NOT EXISTS node_fts_ai AFTER INSERT ON nodes BEGIN
1006
+ INSERT INTO node_fts(node_id, title, summary, metadata)
1007
+ VALUES (new.id, new.title, COALESCE(new.summary, ''), new.metadata_json);
1008
+ END;
1009
+ CREATE TRIGGER IF NOT EXISTS node_fts_au AFTER UPDATE ON nodes BEGIN
1010
+ DELETE FROM node_fts WHERE node_id = old.id;
1011
+ INSERT INTO node_fts(node_id, title, summary, metadata)
1012
+ VALUES (new.id, new.title, COALESCE(new.summary, ''), new.metadata_json);
1013
+ END;
1014
+ CREATE TRIGGER IF NOT EXISTS node_fts_ad AFTER DELETE ON nodes BEGIN
1015
+ DELETE FROM node_fts WHERE node_id = old.id;
1016
+ END;
1017
+ """
1018
+
1019
+ def _init_fts(self) -> None:
1020
+ self._fts_enabled = False
1021
+ try:
1022
+ with self._connect() as conn:
1023
+ conn.executescript(self._FTS_SQL)
1024
+ fts_count = conn.execute("SELECT count(*) AS c FROM node_fts").fetchone()["c"]
1025
+ if fts_count == 0:
1026
+ conn.execute(
1027
+ "INSERT INTO node_fts(node_id, title, summary, metadata) "
1028
+ "SELECT id, title, COALESCE(summary, ''), metadata_json FROM nodes"
1029
+ )
1030
+ self._fts_enabled = True
1031
+ except sqlite3.OperationalError as exc:
1032
+ # FTS5/trigram not compiled into this SQLite build. LIKE search
1033
+ # stays authoritative; the capability is reported, never faked.
1034
+ logging.info("FTS5 trigram index unavailable (%s); keyword search uses LIKE scans.", exc)
1035
+
1036
+ def _fts_match_ids(self, conn: sqlite3.Connection, query: str, limit: int) -> List[str]:
1037
+ """Ranked node ids for a trigram FTS query ('' on any failure)."""
1038
+ if not getattr(self, "_fts_enabled", False) or len(query) < 3:
1039
+ return []
1040
+ escaped = query.replace('"', '""')
1041
+ try:
1042
+ rows = conn.execute(
1043
+ 'SELECT node_id FROM node_fts WHERE node_fts MATCH ? ORDER BY rank LIMIT ?',
1044
+ (f'"{escaped}"', limit),
1045
+ ).fetchall()
1046
+ except sqlite3.OperationalError:
1047
+ return []
1048
+ return [row["node_id"] for row in rows]
993
1049
 
994
1050
  # SQL views that reconstruct the *exact* legacy row shape on top of the
995
1051
  # normalized v2 tables, so the read methods run unchanged against either
@@ -1128,26 +1184,40 @@ class KnowledgeGraphStore:
1128
1184
  self, conn: sqlite3.Connection, node_id: str, node_type: str, title: str,
1129
1185
  summary: Optional[str], metadata_json: Optional[str],
1130
1186
  *, created_at: Optional[str] = None, updated_at: Optional[str] = None,
1187
+ owner: Optional[str] = None, workspace_id: Optional[str] = None,
1188
+ visibility: Optional[str] = None,
1131
1189
  ) -> None:
1132
1190
  if KGStoreV2 is None:
1133
1191
  return
1134
1192
  ts = updated_at or _now()
1135
1193
  norm_type = NodeType.from_legacy(node_type).value if NodeType is not None else node_type
1194
+ # Scope resolution: explicit param > metadata hints > legacy-global.
1195
+ # 'legacy' (not 'private') marks unscoped rows — the column default
1196
+ # must never silently privatize previously machine-shared data.
1197
+ meta = _safe_loads(metadata_json) if metadata_json else {}
1198
+ owner = owner or meta.get("user_email") or meta.get("owner") or None
1199
+ workspace_id = workspace_id or meta.get("workspace_id") or None
1200
+ visibility = visibility or ("legacy" if workspace_id is None else "workspace")
1136
1201
  try:
1137
1202
  conn.execute(
1138
1203
  """
1139
1204
  INSERT INTO nodes_v2(id, type, legacy_type, label, summary, attrs,
1140
- owner_id, visibility, created_at, updated_at,
1141
- importance_score)
1142
- VALUES (?, ?, ?, ?, ?, ?, NULL, 'private', ?, ?, 0.0)
1205
+ owner_id, workspace_id, visibility,
1206
+ created_at, updated_at, importance_score)
1207
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0.0)
1143
1208
  ON CONFLICT(id) DO UPDATE SET
1144
1209
  type=excluded.type, legacy_type=excluded.legacy_type,
1145
1210
  label=excluded.label, summary=excluded.summary,
1146
- attrs=excluded.attrs, updated_at=excluded.updated_at
1211
+ attrs=excluded.attrs, updated_at=excluded.updated_at,
1212
+ owner_id=COALESCE(excluded.owner_id, nodes_v2.owner_id),
1213
+ workspace_id=COALESCE(excluded.workspace_id, nodes_v2.workspace_id),
1214
+ visibility=CASE WHEN excluded.visibility != 'legacy'
1215
+ THEN excluded.visibility
1216
+ ELSE nodes_v2.visibility END
1147
1217
  """,
1148
1218
  (node_id, norm_type, node_type, title, summary,
1149
1219
  metadata_json if metadata_json is not None else "{}",
1150
- created_at or ts, ts),
1220
+ owner, workspace_id, visibility, created_at or ts, ts),
1151
1221
  )
1152
1222
  except Exception as ex:
1153
1223
  logging.debug("knowledge_graph: v2 node projection skipped (%s): %s", node_id, ex)
@@ -1169,8 +1239,7 @@ class KnowledgeGraphStore:
1169
1239
  INSERT INTO edges_v2(id, source, target, type, legacy_type, weight,
1170
1240
  confidence, evidence, metadata, created_by, created_at)
1171
1241
  VALUES (?, ?, ?, ?, ?, ?, ?, '[]', ?, 'legacy', ?)
1172
- ON CONFLICT(source, target, legacy_type) DO UPDATE SET
1173
- type=excluded.type,
1242
+ ON CONFLICT(source, target, type, legacy_type) DO UPDATE SET
1174
1243
  weight=max(edges_v2.weight, excluded.weight),
1175
1244
  confidence=excluded.confidence,
1176
1245
  metadata=excluded.metadata
@@ -1178,9 +1247,125 @@ class KnowledgeGraphStore:
1178
1247
  (eid, from_node, to_node, norm_type, edge_type, float(weight),
1179
1248
  confidence, meta_str, created_at or _now()),
1180
1249
  )
1250
+ # Temporal record: every observation of this relationship is kept
1251
+ # (the UNIQUE upsert + weight=max alone would erase recurrence).
1252
+ row = conn.execute(
1253
+ "SELECT id FROM edges_v2 WHERE source=? AND target=? AND type=? AND legacy_type=?",
1254
+ (from_node, to_node, norm_type, edge_type),
1255
+ ).fetchone()
1256
+ if row is not None:
1257
+ conn.execute(
1258
+ "INSERT INTO edge_occurrences(edge_id, observed_at, weight, source) VALUES (?, ?, ?, ?)",
1259
+ (row["id"], created_at or _now(), float(weight),
1260
+ _safe_loads(meta_str).get("source")),
1261
+ )
1181
1262
  except Exception as ex:
1182
1263
  logging.debug("knowledge_graph: v2 edge projection skipped (%s->%s): %s", from_node, to_node, ex)
1183
1264
 
1265
+ def curate(self, *, max_documents: int = 200, max_new_nodes: int = 8) -> Dict[str, Any]:
1266
+ """On-demand graph curation (T4.4 — graph_curator goes live).
1267
+
1268
+ Runs the curator's gated topic-promotion pipeline over recent content
1269
+ nodes: candidates are clustered, secret-bearing labels are refused,
1270
+ and only multi-source topics above the importance threshold become
1271
+ Topic nodes (with MENTIONS edges back to their sources and a real
1272
+ importance_score in nodes_v2). Explicit and observable — the result
1273
+ reports everything promoted AND everything skipped, with reasons.
1274
+ """
1275
+ from latticeai.core.graph_curator import auto_build_graph_overlay
1276
+
1277
+ content_types = (
1278
+ "Document", "File", "CodeFile", "Message", "AIResponse",
1279
+ "Chat", "Page", "Slide", "Spreadsheet",
1280
+ )
1281
+ nt, _ = self._read_tables()
1282
+ with self._connect() as conn:
1283
+ placeholders = ",".join("?" for _ in content_types)
1284
+ rows = conn.execute(
1285
+ f"""
1286
+ SELECT id, type, title, summary FROM {nt}
1287
+ WHERE type IN ({placeholders})
1288
+ ORDER BY updated_at DESC, id ASC LIMIT ?
1289
+ """,
1290
+ (*content_types, max(1, min(int(max_documents), 2000))),
1291
+ ).fetchall()
1292
+ existing_labels = {
1293
+ str(row["title"] or "").strip().lower()
1294
+ for row in conn.execute(
1295
+ f"SELECT title FROM {nt} WHERE type IN ('Topic', 'Concept')"
1296
+ ).fetchall()
1297
+ }
1298
+ documents = [
1299
+ {
1300
+ "id": row["id"],
1301
+ "text": f"{row['title']} {row['summary'] or ''}",
1302
+ "kind": "file" if row["type"] in {"Document", "File", "CodeFile", "Spreadsheet"} else "chat",
1303
+ }
1304
+ for row in rows
1305
+ ]
1306
+ overlay = auto_build_graph_overlay(
1307
+ documents,
1308
+ existing_node_labels=existing_labels,
1309
+ max_new_nodes=max(1, min(int(max_new_nodes), 50)),
1310
+ )
1311
+ promoted: List[Dict[str, Any]] = []
1312
+ with self._connect() as conn:
1313
+ valid_ids = {row["id"] for row in rows}
1314
+ for promo in overlay["promotions"]:
1315
+ topic_id = f"topic:{_slug(promo['label'])}"
1316
+ self._upsert_node(
1317
+ conn, topic_id, "Topic", promo["label"],
1318
+ metadata={
1319
+ "curated": True,
1320
+ "importance": promo["importance"],
1321
+ "aliases": promo["aliases"],
1322
+ "source": "graph_curator",
1323
+ },
1324
+ )
1325
+ conn.execute(
1326
+ "UPDATE nodes_v2 SET importance_score=? WHERE id=?",
1327
+ (float(promo["importance"]), topic_id),
1328
+ )
1329
+ linked = 0
1330
+ for source_id in promo["sources"][:10]:
1331
+ if source_id in valid_ids:
1332
+ self._upsert_edge(
1333
+ conn, source_id, topic_id, "MENTIONS",
1334
+ weight=0.6, metadata={"source": "graph_curator"},
1335
+ )
1336
+ linked += 1
1337
+ promoted.append({
1338
+ "node_id": topic_id,
1339
+ "label": promo["label"],
1340
+ "importance": promo["importance"],
1341
+ "linked_sources": linked,
1342
+ })
1343
+ return {
1344
+ "status": "ok",
1345
+ "documents_scanned": len(documents),
1346
+ "candidates_total": overlay["candidates_total"],
1347
+ "promoted": promoted,
1348
+ "skipped": overlay["skipped"][:50],
1349
+ "skipped_total": len(overlay["skipped"]),
1350
+ }
1351
+
1352
+ def mark_superseded(self, old_node_id: str, new_node_id: str) -> Dict[str, Any]:
1353
+ """Record that ``old_node_id`` was replaced by ``new_node_id``.
1354
+
1355
+ The old node stays queryable (knowledge is durable); readers can follow
1356
+ the revision chain via ``nodes_v2.superseded_by``.
1357
+ """
1358
+ with self._connect() as conn:
1359
+ for node_id in (old_node_id, new_node_id):
1360
+ exists = conn.execute("SELECT 1 FROM nodes_v2 WHERE id=?", (node_id,)).fetchone()
1361
+ if not exists:
1362
+ raise FileNotFoundError(node_id)
1363
+ conn.execute(
1364
+ "UPDATE nodes_v2 SET superseded_by=?, updated_at=? WHERE id=?",
1365
+ (new_node_id, _now(), old_node_id),
1366
+ )
1367
+ return {"status": "ok", "node_id": old_node_id, "superseded_by": new_node_id}
1368
+
1184
1369
  def _v2_delete_nodes(self, conn: sqlite3.Connection, ids) -> None:
1185
1370
  """Mirror legacy node deletions into v2 (edges_v2 cascade on the FK)."""
1186
1371
  if KGStoreV2 is None:
@@ -1241,6 +1426,9 @@ class KnowledgeGraphStore:
1241
1426
  summary: str = "",
1242
1427
  metadata: Optional[Dict[str, Any]] = None,
1243
1428
  raw: Optional[Dict[str, Any]] = None,
1429
+ owner: Optional[str] = None,
1430
+ workspace_id: Optional[str] = None,
1431
+ visibility: Optional[str] = None,
1244
1432
  ) -> str:
1245
1433
  now = _now()
1246
1434
  # Canonical stored values, computed once and shared with the v2
@@ -1263,7 +1451,8 @@ class KnowledgeGraphStore:
1263
1451
  )
1264
1452
  # dual-write: project into the v2 graph on the same transaction
1265
1453
  self._v2_project_node(conn, node_id, node_type, title_s, summary_s, meta_json,
1266
- created_at=now, updated_at=now)
1454
+ created_at=now, updated_at=now,
1455
+ owner=owner, workspace_id=workspace_id, visibility=visibility)
1267
1456
  if node_type != "Chunk":
1268
1457
  self._upsert_vector_item(
1269
1458
  conn,
@@ -1284,6 +1473,16 @@ class KnowledgeGraphStore:
1284
1473
  weight: float = 1.0,
1285
1474
  metadata: Optional[Dict[str, Any]] = None,
1286
1475
  ) -> str:
1476
+ # v4 write door: every new edge stores the canonical EdgeType value —
1477
+ # free-string types (e.g. '포함함', '언급함') are normalized here, so no
1478
+ # caller can mint new legacy taxonomy. The original label survives in
1479
+ # metadata.legacy_label for traceability.
1480
+ if EdgeType is not None:
1481
+ canonical = EdgeType.from_legacy(edge_type).value
1482
+ if canonical != edge_type:
1483
+ metadata = dict(metadata or {})
1484
+ metadata.setdefault("legacy_label", edge_type)
1485
+ edge_type = canonical
1287
1486
  edge_id = f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}')[:24]}"
1288
1487
  now = _now()
1289
1488
  meta_json = _json(metadata) # canonical string shared with the projection
@@ -3189,6 +3388,41 @@ class KnowledgeGraphStore:
3189
3388
  ).fetchall()
3190
3389
  return {"items": [self._provenance_row(r) for r in rows], "count": len(rows)}
3191
3390
 
3391
+ def provenance_coverage(self) -> Dict[str, Any]:
3392
+ """How much of the brain is explainable: nodes with vs without
3393
+ provenance, per node type — the honesty metric for 'every source goes
3394
+ through the pipeline'. Pre-v4 nodes ingested before provenance existed
3395
+ legitimately count as uncovered."""
3396
+ nt, _ = self._read_tables()
3397
+ with self._connect() as conn:
3398
+ total = conn.execute(f"SELECT COUNT(*) FROM {nt}").fetchone()[0]
3399
+ covered = conn.execute(
3400
+ f"SELECT COUNT(*) FROM {nt} WHERE id IN (SELECT DISTINCT node_id FROM ingestion_provenance)"
3401
+ ).fetchone()[0]
3402
+ uncovered_by_type = {
3403
+ row["type"]: row["c"]
3404
+ for row in conn.execute(
3405
+ f"""
3406
+ SELECT type, COUNT(*) AS c FROM {nt}
3407
+ WHERE id NOT IN (SELECT DISTINCT node_id FROM ingestion_provenance)
3408
+ GROUP BY type ORDER BY c DESC LIMIT 20
3409
+ """
3410
+ ).fetchall()
3411
+ }
3412
+ by_source = {
3413
+ row["source_type"]: row["c"]
3414
+ for row in conn.execute(
3415
+ "SELECT source_type, COUNT(*) AS c FROM ingestion_provenance GROUP BY source_type"
3416
+ ).fetchall()
3417
+ }
3418
+ return {
3419
+ "total_nodes": total,
3420
+ "nodes_with_provenance": covered,
3421
+ "coverage_ratio": round(covered / total, 4) if total else None,
3422
+ "uncovered_by_type": uncovered_by_type,
3423
+ "provenance_by_source_type": by_source,
3424
+ }
3425
+
3192
3426
  def provenance_stats(self) -> Dict[str, Any]:
3193
3427
  """Aggregate provenance counts for the Knowledge Graph status surface."""
3194
3428
  with self._connect() as conn:
@@ -3230,23 +3464,52 @@ class KnowledgeGraphStore:
3230
3464
  "embed_dim": _EMBED_DIM,
3231
3465
  }
3232
3466
 
3233
- def export_graph_data(self) -> Dict[str, Any]:
3467
+ def export_graph_data(self, *, workspace_id: Optional[str] = None) -> Dict[str, Any]:
3234
3468
  """Raw, lossless logical export of the graph (nodes/edges/chunks/sources/
3235
3469
  provenance). Vector embeddings are intentionally omitted — they are
3236
3470
  re-derived on import — so the artifact stays portable and small. Use
3237
3471
  :meth:`backup_database` for a faithful binary copy incl. embeddings.
3472
+
3473
+ ``workspace_id`` REALLY filters (v4): the artifact contains only nodes
3474
+ scoped to that workspace plus legacy-global rows (NULL scope, readable
3475
+ machine-wide by definition), with edges/chunks/provenance restricted to
3476
+ the surviving nodes. Pre-v4 this parameter was stamped into the header
3477
+ while the data exported everything — a header that lied.
3238
3478
  """
3239
3479
  with self._connect() as conn:
3240
3480
  def rows(table: str):
3241
3481
  return [dict(r) for r in conn.execute(f"SELECT * FROM {table}").fetchall()]
3242
3482
 
3243
- data = {
3244
- "nodes": rows("nodes"),
3245
- "edges": rows("edges"),
3246
- "chunks": rows("chunks"),
3247
- "knowledge_sources": rows("knowledge_sources"),
3248
- "provenance": rows("ingestion_provenance"),
3249
- }
3483
+ if workspace_id:
3484
+ keep_ids = {
3485
+ row["id"]
3486
+ for row in conn.execute(
3487
+ "SELECT id FROM nodes_v2 WHERE workspace_id = ? OR workspace_id IS NULL",
3488
+ (workspace_id,),
3489
+ ).fetchall()
3490
+ }
3491
+ nodes = [n for n in rows("nodes") if n["id"] in keep_ids]
3492
+ edges = [
3493
+ e for e in rows("edges")
3494
+ if e["from_node"] in keep_ids and e["to_node"] in keep_ids
3495
+ ]
3496
+ chunks = [c for c in rows("chunks") if c["source_node"] in keep_ids]
3497
+ provenance = [p for p in rows("ingestion_provenance") if p["node_id"] in keep_ids]
3498
+ data = {
3499
+ "nodes": nodes,
3500
+ "edges": edges,
3501
+ "chunks": chunks,
3502
+ "knowledge_sources": rows("knowledge_sources"),
3503
+ "provenance": provenance,
3504
+ }
3505
+ else:
3506
+ data = {
3507
+ "nodes": rows("nodes"),
3508
+ "edges": rows("edges"),
3509
+ "chunks": rows("chunks"),
3510
+ "knowledge_sources": rows("knowledge_sources"),
3511
+ "provenance": rows("ingestion_provenance"),
3512
+ }
3250
3513
  data["counts"] = {k: len(v) for k, v in data.items()}
3251
3514
  return data
3252
3515
 
@@ -3610,7 +3873,40 @@ class KnowledgeGraphStore:
3610
3873
  "generated_at": datetime.now().isoformat(timespec="seconds"),
3611
3874
  }
3612
3875
 
3613
- def graph(self, limit: int = 300) -> Dict[str, Any]:
3876
+ def workspaces_of(self, node_ids) -> Dict[str, Optional[str]]:
3877
+ """Map node ids to their workspace scope (None = legacy-global)."""
3878
+ ids = [str(i) for i in node_ids if i]
3879
+ if not ids:
3880
+ return {}
3881
+ placeholders = ",".join("?" for _ in ids)
3882
+ with self._connect() as conn:
3883
+ try:
3884
+ return {
3885
+ row["id"]: row["workspace_id"]
3886
+ for row in conn.execute(
3887
+ f"SELECT id, workspace_id FROM nodes_v2 WHERE id IN ({placeholders})", ids
3888
+ ).fetchall()
3889
+ }
3890
+ except Exception:
3891
+ return {}
3892
+
3893
+ def filter_scoped_nodes(self, items, allowed_workspaces, *, id_key: str = "id"):
3894
+ """Drop items scoped to a workspace the caller is not a member of.
3895
+
3896
+ ``allowed_workspaces=None`` means no scoping (single-user / no-auth
3897
+ mode). Legacy-global rows (no workspace) stay visible to everyone on
3898
+ the machine — the documented pre-v4 compatibility behavior.
3899
+ """
3900
+ if allowed_workspaces is None:
3901
+ return list(items)
3902
+ allowed = set(allowed_workspaces)
3903
+ scopes = self.workspaces_of([item.get(id_key) for item in items])
3904
+ return [
3905
+ item for item in items
3906
+ if scopes.get(item.get(id_key)) is None or scopes.get(item.get(id_key)) in allowed
3907
+ ]
3908
+
3909
+ def graph(self, limit: int = 300, *, allowed_workspaces=None) -> Dict[str, Any]:
3614
3910
  limit = max(1, min(int(limit or 300), 2000))
3615
3911
  visible = ",".join(f"'{t}'" for t in self._GRAPH_VISIBLE_TYPES)
3616
3912
  nt, et = self._read_tables()
@@ -3660,6 +3956,11 @@ class KnowledgeGraphStore:
3660
3956
  for row in edge_rows
3661
3957
  ]
3662
3958
 
3959
+ if allowed_workspaces is not None:
3960
+ nodes = self.filter_scoped_nodes(nodes, allowed_workspaces)
3961
+ kept_ids = {node["id"] for node in nodes}
3962
+ edges = [e for e in edges if e["from"] in kept_ids and e["to"] in kept_ids]
3963
+
3663
3964
  degree_map: Dict[str, int] = {}
3664
3965
  now = datetime.now()
3665
3966
  node_by_id = {node["id"]: node for node in nodes}
@@ -3735,16 +4036,32 @@ class KnowledgeGraphStore:
3735
4036
  with self._connect() as conn:
3736
4037
  rows = []
3737
4038
  if query:
3738
- rows = conn.execute(
3739
- f"""
3740
- SELECT id, type, title, summary, metadata_json, updated_at
3741
- FROM {nt}
3742
- WHERE title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?
3743
- ORDER BY updated_at DESC, id ASC
3744
- LIMIT ?
3745
- """,
3746
- (q, q, q, limit),
3747
- ).fetchall()
4039
+ fts_ids = self._fts_match_ids(conn, query, limit)
4040
+ if fts_ids:
4041
+ placeholders = ",".join("?" for _ in fts_ids)
4042
+ by_id = {
4043
+ row["id"]: row
4044
+ for row in conn.execute(
4045
+ f"""
4046
+ SELECT id, type, title, summary, metadata_json, updated_at
4047
+ FROM {nt} WHERE id IN ({placeholders})
4048
+ """,
4049
+ fts_ids,
4050
+ ).fetchall()
4051
+ }
4052
+ # Preserve FTS bm25 rank order.
4053
+ rows = [by_id[i] for i in fts_ids if i in by_id]
4054
+ else:
4055
+ rows = conn.execute(
4056
+ f"""
4057
+ SELECT id, type, title, summary, metadata_json, updated_at
4058
+ FROM {nt}
4059
+ WHERE title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?
4060
+ ORDER BY updated_at DESC, id ASC
4061
+ LIMIT ?
4062
+ """,
4063
+ (q, q, q, limit),
4064
+ ).fetchall()
3748
4065
 
3749
4066
  if len(rows) < limit:
3750
4067
  terms = _topic_candidates(query, limit=8)
@@ -3779,6 +4096,10 @@ class KnowledgeGraphStore:
3779
4096
  } else 0
3780
4097
  return (hits, type_boost, row["updated_at"] or "")
3781
4098
 
4099
+ # Deterministic contract: rows with equal relevance order by id ASC
4100
+ # (stable sort preserves the pre-sort under reverse=True), matching
4101
+ # the legacy LIKE path regardless of FTS bm25 tie ordering.
4102
+ rows = sorted(rows, key=lambda r: r["id"])
3782
4103
  rows = sorted(rows, key=score, reverse=True)[:limit]
3783
4104
  return {
3784
4105
  "query": query,
@@ -4263,6 +4584,9 @@ class KnowledgeGraphStore:
4263
4584
  "backend": "sqlite",
4264
4585
  "embedding_model": self._embedding_model.model_id,
4265
4586
  "embedding_dim": self._embedding_model.dim,
4587
+ # Honest capability report: trigram FTS5 keyword index, or
4588
+ # LIKE-scan fallback when this SQLite build lacks it.
4589
+ "fts_enabled": bool(getattr(self, "_fts_enabled", False)),
4266
4590
  },
4267
4591
  "source_items": len(source_items),
4268
4592
  "indexed_items": sum(vector_counts.values()),
@@ -4366,21 +4690,26 @@ class KnowledgeGraphStore:
4366
4690
  return {"status": "skipped", "removed_nodes": 0}
4367
4691
  conv_id = f"conversation:{_slug(conversation_id)}"
4368
4692
  with self._connect() as conn:
4693
+ # Edge rows may carry the legacy lowercase label (pre-v4) or the
4694
+ # canonical EdgeType value (v4 write door) — match both.
4369
4695
  direct_ids = [
4370
4696
  row["to_node"]
4371
4697
  for row in conn.execute(
4372
- "SELECT to_node FROM edges WHERE from_node=? AND type='contains'",
4698
+ "SELECT to_node FROM edges WHERE from_node=? AND type IN ('contains', 'CONTAINS')",
4373
4699
  (conv_id,),
4374
4700
  )
4375
4701
  ]
4376
4702
  remove_ids = set(direct_ids)
4703
+ child_types = [
4704
+ "has_chunk", "implies", "contains_signal", "has_page",
4705
+ "has_slide", "has_sheet", "contains_image",
4706
+ ]
4707
+ child_types += [t.upper() for t in child_types]
4708
+ placeholders = ",".join("?" for _ in child_types)
4377
4709
  for source_id in list(direct_ids):
4378
4710
  for row in conn.execute(
4379
- """
4380
- SELECT to_node FROM edges
4381
- WHERE from_node=? AND type IN ('has_chunk', 'implies', 'contains_signal', 'has_page', 'has_slide', 'has_sheet', 'contains_image')
4382
- """,
4383
- (source_id,),
4711
+ f"SELECT to_node FROM edges WHERE from_node=? AND type IN ({placeholders})",
4712
+ (source_id, *child_types),
4384
4713
  ):
4385
4714
  remove_ids.add(row["to_node"])
4386
4715
  remove_ids.add(conv_id)
@@ -1,130 +1,14 @@
1
- """Knowledge graph page and API routes."""
1
+ """Deprecation shim — the knowledge graph router moved in v4.
2
2
 
3
- from pathlib import Path
4
- from typing import Any, Callable, Dict, Optional
3
+ The ``/knowledge-graph/*`` data router (and legacy ``/graph`` page routes)
4
+ now live in :mod:`latticeai.api.knowledge_graph`. This root module remains
5
+ importable for the deprecation window and will be removed in a future major
6
+ release.
7
+ """
5
8
 
6
- from fastapi import APIRouter, HTTPException, Request
7
- from fastapi.responses import FileResponse
8
- from pydantic import BaseModel
9
+ from latticeai.api.knowledge_graph import ( # noqa: F401
10
+ KnowledgeGraphIngestRequest,
11
+ create_knowledge_graph_router,
12
+ )
9
13
 
10
-
11
- class KnowledgeGraphIngestRequest(BaseModel):
12
- type: str
13
- content: str = ""
14
- role: Optional[str] = None
15
- title: Optional[str] = None
16
- source: Optional[str] = None
17
- conversation_id: Optional[str] = None
18
- user_email: Optional[str] = None
19
- user_nickname: Optional[str] = None
20
- metadata: Optional[Dict[str, Any]] = None
21
-
22
-
23
- def create_knowledge_graph_router(
24
- *,
25
- get_graph: Callable[[], Any],
26
- require_graph: Callable[[], None],
27
- require_user: Callable[[Request], str],
28
- static_dir: Path,
29
- ) -> APIRouter:
30
- router = APIRouter()
31
-
32
- def graph():
33
- require_graph()
34
- return get_graph()
35
-
36
- @router.get("/graph")
37
- async def knowledge_graph_page(request: Request):
38
- """Serve the interactive knowledge graph canvas UI."""
39
- graph()
40
- require_user(request)
41
- response = FileResponse(static_dir / "graph.html")
42
- response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
43
- response.headers["Pragma"] = "no-cache"
44
- response.headers["Expires"] = "0"
45
- return response
46
-
47
- @router.get("/knowledge-graph")
48
- async def knowledge_graph_legacy_page(request: Request):
49
- """Backward-compatible route for the graph page."""
50
- graph()
51
- require_user(request)
52
- response = FileResponse(static_dir / "graph.html")
53
- response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
54
- response.headers["Pragma"] = "no-cache"
55
- response.headers["Expires"] = "0"
56
- return response
57
-
58
- @router.get("/knowledge-graph/stats")
59
- async def knowledge_graph_stats(request: Request):
60
- require_user(request)
61
- return graph().stats()
62
-
63
- @router.get("/knowledge-graph/schema")
64
- async def knowledge_graph_schema(request: Request):
65
- require_user(request)
66
- stats = graph().stats()
67
- return {
68
- "legacy_schema_version": stats.get("schema_version"),
69
- "v2_schema_available": stats.get("v2_schema_available"),
70
- "v2": stats.get("v2"),
71
- }
72
-
73
- @router.get("/knowledge-graph/graph")
74
- async def knowledge_graph_data(request: Request, limit: int = 300):
75
- require_user(request)
76
- return graph().graph(limit)
77
-
78
- @router.get("/knowledge-graph/documents")
79
- async def knowledge_graph_documents(request: Request, limit: int = 200):
80
- """Ingested documents (uploads + indexed local docs) with index state.
81
-
82
- Backs the Files view so uploaded content is visible end-to-end:
83
- upload → Files → Knowledge Graph → Hybrid Search → Chat.
84
- """
85
- require_user(request)
86
- return graph().list_documents(limit)
87
-
88
- @router.get("/knowledge-graph/search")
89
- async def knowledge_graph_search(q: str, request: Request, limit: int = 30):
90
- require_user(request)
91
- if not q or not q.strip():
92
- return {"query": q, "matches": []}
93
- return graph().search(q, limit)
94
-
95
- @router.get("/knowledge-graph/context")
96
- async def knowledge_graph_context(q: str, request: Request, limit: int = 6):
97
- require_user(request)
98
- return {"query": q, "context": graph().context_for_query(q, limit)}
99
-
100
- @router.get("/knowledge-graph/neighbors/{node_id:path}")
101
- async def knowledge_graph_neighbors(node_id: str, request: Request):
102
- require_user(request)
103
- if not node_id:
104
- raise HTTPException(status_code=400, detail="node_id required")
105
- return graph().neighbors(node_id)
106
-
107
- @router.post("/knowledge-graph/ingest")
108
- async def knowledge_graph_ingest(req: KnowledgeGraphIngestRequest, request: Request):
109
- current_user = require_user(request)
110
- kg = graph()
111
- event_type = (req.type or "").strip().lower()
112
- if event_type not in {"message", "ai_response", "note"}:
113
- raise HTTPException(status_code=400, detail="지원하는 type: message, ai_response, note")
114
- role = req.role or ("assistant" if event_type == "ai_response" else "user")
115
- return kg.ingest_message(
116
- role,
117
- req.content,
118
- user_email=req.user_email or current_user,
119
- user_nickname=req.user_nickname,
120
- source=req.source or "mcp",
121
- conversation_id=req.conversation_id,
122
- raw={
123
- "type": req.type,
124
- "title": req.title,
125
- "content": req.content,
126
- "metadata": req.metadata or {},
127
- },
128
- )
129
-
130
- return router
14
+ __all__ = ["KnowledgeGraphIngestRequest", "create_knowledge_graph_router"]