ltcai 8.8.0 → 9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +33 -37
  2. package/auto_setup.py +84 -70
  3. package/docs/CHANGELOG.md +113 -237
  4. package/docs/CODE_REVIEW_2026-07-06.md +764 -0
  5. package/docs/COMMUNITY_AND_PLUGINS.md +3 -3
  6. package/docs/DEVELOPMENT.md +10 -10
  7. package/docs/LEGACY_COMPATIBILITY.md +1 -1
  8. package/docs/ONBOARDING.md +2 -2
  9. package/docs/PRODUCT_DIRECTION_REVIEW.md +3 -2
  10. package/docs/ROADMAP_RECOMMENDATIONS.md +61 -36
  11. package/docs/TRUST_MODEL.md +5 -1
  12. package/docs/WHY_LATTICE.md +4 -3
  13. package/docs/architecture.md +4 -0
  14. package/docs/kg-schema.md +1 -1
  15. package/lattice_brain/__init__.py +1 -1
  16. package/lattice_brain/archive.py +4 -9
  17. package/lattice_brain/conversations.py +156 -21
  18. package/lattice_brain/embeddings.py +38 -2
  19. package/lattice_brain/graph/_kg_common.py +39 -496
  20. package/lattice_brain/graph/_kg_constants.py +243 -0
  21. package/lattice_brain/graph/_kg_fsutil.py +297 -0
  22. package/lattice_brain/graph/discovery.py +0 -948
  23. package/lattice_brain/graph/discovery_index.py +972 -0
  24. package/lattice_brain/graph/json_utils.py +25 -0
  25. package/lattice_brain/graph/retrieval.py +66 -597
  26. package/lattice_brain/graph/retrieval_docgen.py +210 -0
  27. package/lattice_brain/graph/retrieval_vector.py +460 -0
  28. package/lattice_brain/graph/runtime.py +16 -0
  29. package/lattice_brain/graph/store.py +6 -0
  30. package/lattice_brain/ingestion.py +68 -0
  31. package/lattice_brain/portability.py +1 -9
  32. package/lattice_brain/quality.py +98 -4
  33. package/lattice_brain/runtime/agent_runtime.py +166 -0
  34. package/lattice_brain/runtime/multi_agent.py +1 -1
  35. package/lattice_brain/utils.py +28 -0
  36. package/latticeai/__init__.py +1 -1
  37. package/latticeai/api/chat.py +368 -418
  38. package/latticeai/api/chat_helpers.py +227 -0
  39. package/latticeai/api/computer_use.py +149 -31
  40. package/latticeai/api/marketplace.py +11 -0
  41. package/latticeai/api/mcp.py +3 -2
  42. package/latticeai/api/models.py +4 -1
  43. package/latticeai/api/permissions.py +72 -33
  44. package/latticeai/api/setup.py +17 -2
  45. package/latticeai/api/tools.py +105 -62
  46. package/latticeai/app_factory.py +101 -296
  47. package/latticeai/core/agent.py +25 -7
  48. package/latticeai/core/io_utils.py +37 -0
  49. package/latticeai/core/legacy_compatibility.py +1 -1
  50. package/latticeai/core/local_embeddings.py +2 -4
  51. package/latticeai/core/marketplace.py +33 -2
  52. package/latticeai/core/mcp_catalog.py +450 -0
  53. package/latticeai/core/mcp_registry.py +2 -441
  54. package/latticeai/core/sessions.py +11 -3
  55. package/latticeai/core/tool_registry.py +15 -4
  56. package/latticeai/core/users.py +4 -9
  57. package/latticeai/core/workspace_os.py +1 -1
  58. package/latticeai/core/workspace_os_utils.py +3 -17
  59. package/latticeai/integrations/telegram_bot.py +7 -2
  60. package/latticeai/models/model_providers.py +111 -0
  61. package/latticeai/models/router.py +58 -136
  62. package/latticeai/runtime/audit_runtime.py +27 -16
  63. package/latticeai/runtime/automation_runtime.py +9 -0
  64. package/latticeai/runtime/bootstrap.py +1 -1
  65. package/latticeai/runtime/history_runtime.py +163 -0
  66. package/latticeai/runtime/namespace_runtime.py +173 -0
  67. package/latticeai/runtime/network_config_runtime.py +56 -0
  68. package/latticeai/runtime/sso_config_runtime.py +128 -0
  69. package/latticeai/runtime/user_key_runtime.py +106 -0
  70. package/latticeai/services/app_context.py +1 -0
  71. package/latticeai/services/architecture_readiness.py +2 -2
  72. package/latticeai/services/memory_service.py +213 -0
  73. package/latticeai/services/model_engines.py +79 -12
  74. package/latticeai/services/model_runtime.py +24 -4
  75. package/latticeai/services/platform_runtime.py +9 -1
  76. package/latticeai/services/process_audit.py +208 -0
  77. package/latticeai/services/product_readiness.py +11 -11
  78. package/latticeai/services/review_queue.py +64 -11
  79. package/latticeai/services/run_executor.py +21 -0
  80. package/latticeai/services/search_service.py +106 -30
  81. package/latticeai/services/setup_detection.py +80 -0
  82. package/latticeai/services/tool_dispatch.py +66 -0
  83. package/latticeai/services/workspace_service.py +15 -0
  84. package/package.json +1 -1
  85. package/scripts/check_i18n_literals.mjs +20 -8
  86. package/scripts/i18n_literal_allowlist.json +34 -0
  87. package/scripts/lint_frontend.mjs +6 -2
  88. package/setup_wizard.py +196 -74
  89. package/src-tauri/Cargo.lock +1 -1
  90. package/src-tauri/Cargo.toml +1 -1
  91. package/src-tauri/tauri.conf.json +1 -1
  92. package/static/app/asset-manifest.json +11 -11
  93. package/static/app/assets/{Act-C7K9wsO9.js → Act-21lIXx2E.js} +1 -1
  94. package/static/app/assets/Brain-BqUd5UJJ.js +321 -0
  95. package/static/app/assets/{Capture-B3V4_5Xp.js → Capture-BA7Z2Q1u.js} +1 -1
  96. package/static/app/assets/{Library-Cgj-EF50.js → Library-bFMtyni3.js} +1 -1
  97. package/static/app/assets/System-K6krGCqn.js +1 -0
  98. package/static/app/assets/index-C4R3ws30.js +17 -0
  99. package/static/app/assets/index-ChSeOB02.css +2 -0
  100. package/static/app/assets/primitives-sQU3it5I.js +1 -0
  101. package/static/app/assets/{textarea-CVQkN2Tk.js → textarea-DK3Fd_lR.js} +1 -1
  102. package/static/app/index.html +2 -2
  103. package/static/css/tokens.css +4 -2
  104. package/static/sw.js +1 -1
  105. package/tools/local_files.py +6 -0
  106. package/latticeai/runtime/sso_runtime.py +0 -52
  107. package/static/app/assets/Brain-I1OSzxJu.js +0 -321
  108. package/static/app/assets/System-D1Lkei3I.js +0 -1
  109. package/static/app/assets/index--P0ksosz.js +0 -17
  110. package/static/app/assets/index-DCh5AoXt.css +0 -2
  111. package/static/app/assets/primitives-BLqaKk5g.js +0 -1
@@ -55,9 +55,18 @@ class KnowledgeGraphRetrievalMixin:
55
55
  extracted = meta.get("extracted") or {}
56
56
  node_id = row["id"]
57
57
  chunk_count = conn.execute(
58
- f"SELECT COUNT(*) AS c FROM {nt} WHERE type='Chunk' AND metadata_json LIKE ?",
59
- (f"%{node_id}%",),
58
+ "SELECT COUNT(*) AS c FROM chunks WHERE source_node=?",
59
+ (node_id,),
60
60
  ).fetchone()["c"]
61
+ if not chunk_count:
62
+ # Legacy projections represented chunks as graph nodes and
63
+ # linked them only through metadata_json. Keep read
64
+ # compatibility without making the fragile LIKE path the
65
+ # primary query.
66
+ chunk_count = conn.execute(
67
+ f"SELECT COUNT(*) AS c FROM {nt} WHERE type='Chunk' AND metadata_json LIKE ?",
68
+ (f"%{node_id}%",),
69
+ ).fetchone()["c"]
61
70
  documents.append(
62
71
  {
63
72
  "id": node_id,
@@ -255,7 +264,7 @@ class KnowledgeGraphRetrievalMixin:
255
264
  node.pop("_raw_importance", None)
256
265
  return {"nodes": nodes, "edges": edges}
257
266
 
258
- def search(self, query: str, limit: int = 30) -> Dict[str, Any]:
267
+ def search(self, query: str, limit: int = 30, *, allowed_workspaces=None) -> Dict[str, Any]:
259
268
  query = str(query or "").strip()
260
269
  q = f"%{query}%"
261
270
  limit = max(1, min(int(limit or 30), 100))
@@ -347,9 +356,7 @@ class KnowledgeGraphRetrievalMixin:
347
356
  # the legacy LIKE path regardless of FTS bm25 tie ordering.
348
357
  rows = sorted(rows, key=lambda r: r["id"])
349
358
  rows = sorted(rows, key=score, reverse=True)[:limit]
350
- return {
351
- "query": query,
352
- "matches": [
359
+ matches = [
353
360
  {
354
361
  "id": row["id"],
355
362
  "type": row["type"],
@@ -359,15 +366,17 @@ class KnowledgeGraphRetrievalMixin:
359
366
  "updated_at": row["updated_at"],
360
367
  }
361
368
  for row in rows
362
- ],
363
- }
369
+ ]
370
+ if allowed_workspaces is not None:
371
+ matches = self.filter_scoped_nodes(matches, allowed_workspaces)
372
+ return {"query": query, "matches": matches}
364
373
 
365
- def context_for_query(self, query: str, limit: int = 6) -> str:
374
+ def context_for_query(self, query: str, limit: int = 6, *, allowed_workspaces=None) -> str:
366
375
  """Return compact graph-backed RAG context for chat generation."""
367
376
  query = str(query or "").strip()
368
377
  if not query:
369
378
  return ""
370
- matches = self.search(query, limit).get("matches", [])
379
+ matches = self.search(query, limit, allowed_workspaces=allowed_workspaces).get("matches", [])
371
380
  if not matches:
372
381
  topics = _topic_candidates(query, limit=4)
373
382
  if topics:
@@ -404,6 +413,8 @@ class KnowledgeGraphRetrievalMixin:
404
413
  )
405
414
  if len(matches) >= limit:
406
415
  break
416
+ if allowed_workspaces is not None:
417
+ matches = self.filter_scoped_nodes(matches, allowed_workspaces)
407
418
  lines = []
408
419
  for match in matches[:limit]:
409
420
  meta = match.get("metadata") or {}
@@ -420,8 +431,10 @@ class KnowledgeGraphRetrievalMixin:
420
431
  )
421
432
  return "\n".join(lines)
422
433
 
423
- def neighbors(self, node_id: str) -> Dict[str, Any]:
434
+ def neighbors(self, node_id: str, *, allowed_workspaces=None) -> Dict[str, Any]:
424
435
  """Return direct neighbors (1-hop) of a node."""
436
+ if allowed_workspaces is not None and not self.filter_scoped_nodes([{"id": node_id}], allowed_workspaces):
437
+ raise ValueError(f"graph node not found: {node_id}")
425
438
  nt, et = self._read_tables()
426
439
  with self._connect() as conn:
427
440
  edge_rows = conn.execute(
@@ -458,9 +471,17 @@ class KnowledgeGraphRetrievalMixin:
458
471
  list(neighbor_ids),
459
472
  )
460
473
  ]
474
+ if allowed_workspaces is not None:
475
+ nodes = self.filter_scoped_nodes(nodes, allowed_workspaces)
476
+ kept = {node.get("id") for node in nodes}
477
+ edges = [
478
+ edge for edge in edges
479
+ if (edge.get("from") == node_id or edge.get("from") in kept)
480
+ and (edge.get("to") == node_id or edge.get("to") in kept)
481
+ ]
461
482
  return {"node_id": node_id, "neighbors": nodes, "edges": edges}
462
483
 
463
- def get_node(self, node_id: str) -> Dict[str, Any]:
484
+ def get_node(self, node_id: str, *, allowed_workspaces=None) -> Dict[str, Any]:
464
485
  node_id = str(node_id or "").strip()
465
486
  if not node_id:
466
487
  raise ValueError("node_id required")
@@ -480,7 +501,7 @@ class KnowledgeGraphRetrievalMixin:
480
501
  f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
481
502
  (node_id, node_id),
482
503
  ).fetchone()["c"]
483
- return {
504
+ node = {
484
505
  "id": row["id"],
485
506
  "type": row["type"],
486
507
  "title": row["title"],
@@ -489,6 +510,9 @@ class KnowledgeGraphRetrievalMixin:
489
510
  "updated_at": row["updated_at"],
490
511
  "degree": degree,
491
512
  }
513
+ if allowed_workspaces is not None and not self.filter_scoped_nodes([node], allowed_workspaces):
514
+ raise ValueError(f"graph node not found: {node_id}")
515
+ return node
492
516
 
493
517
  def relationship_search(
494
518
  self,
@@ -497,6 +521,7 @@ class KnowledgeGraphRetrievalMixin:
497
521
  node_id: str = "",
498
522
  relationship_type: str = "",
499
523
  limit: int = 30,
524
+ allowed_workspaces=None,
500
525
  ) -> Dict[str, Any]:
501
526
  query = str(query or "").strip()
502
527
  node_id = str(node_id or "").strip()
@@ -535,11 +560,7 @@ class KnowledgeGraphRetrievalMixin:
535
560
  """,
536
561
  (*params, limit),
537
562
  ).fetchall()
538
- return {
539
- "query": query,
540
- "node_id": node_id,
541
- "relationship_type": relationship_type,
542
- "relationships": [
563
+ relationships = [
543
564
  {
544
565
  "id": row["id"],
545
566
  "type": row["type"],
@@ -562,15 +583,32 @@ class KnowledgeGraphRetrievalMixin:
562
583
  },
563
584
  }
564
585
  for row in rows
565
- ],
586
+ ]
587
+ if allowed_workspaces is not None:
588
+ kept = []
589
+ for rel in relationships:
590
+ endpoints = [
591
+ {"id": (rel.get("source") or {}).get("id")},
592
+ {"id": (rel.get("target") or {}).get("id")},
593
+ ]
594
+ if len(self.filter_scoped_nodes(endpoints, allowed_workspaces)) == 2:
595
+ kept.append(rel)
596
+ relationships = kept
597
+ return {
598
+ "query": query,
599
+ "node_id": node_id,
600
+ "relationship_type": relationship_type,
601
+ "relationships": relationships,
566
602
  }
567
603
 
568
604
  def traverse(
569
- self, node_id: str, *, depth: int = 1, limit: int = 100
605
+ self, node_id: str, *, depth: int = 1, limit: int = 100, allowed_workspaces=None
570
606
  ) -> Dict[str, Any]:
571
607
  node_id = str(node_id or "").strip()
572
608
  if not node_id:
573
609
  raise ValueError("node_id required")
610
+ if allowed_workspaces is not None and not self.filter_scoped_nodes([{"id": node_id}], allowed_workspaces):
611
+ raise ValueError(f"graph node not found: {node_id}")
574
612
  depth = max(0, min(int(depth or 1), 4))
575
613
  limit = max(1, min(int(limit or 100), 500))
576
614
  nt, et = self._read_tables()
@@ -617,10 +655,7 @@ class KnowledgeGraphRetrievalMixin:
617
655
  """,
618
656
  list(visited),
619
657
  ).fetchall()
620
- return {
621
- "root": node_id,
622
- "depth": depth,
623
- "nodes": [
658
+ nodes = [
624
659
  {
625
660
  "id": row["id"],
626
661
  "type": row["type"],
@@ -630,381 +665,13 @@ class KnowledgeGraphRetrievalMixin:
630
665
  "updated_at": row["updated_at"],
631
666
  }
632
667
  for row in node_rows
633
- ],
634
- "edges": list(edges_by_id.values()),
635
- }
636
-
637
- def _iter_vector_source_items(
638
- self,
639
- conn: sqlite3.Connection,
640
- *,
641
- include_nodes: bool = True,
642
- include_chunks: bool = True,
643
- ) -> List[Dict[str, Any]]:
644
- items: List[Dict[str, Any]] = []
645
- if include_nodes:
646
- for row in conn.execute(
647
- """
648
- SELECT id, type, title, summary, metadata_json
649
- FROM nodes
650
- WHERE type <> 'Chunk'
651
- ORDER BY updated_at DESC, id ASC
652
- """
653
- ).fetchall():
654
- metadata = _safe_loads(row["metadata_json"])
655
- text = self._vector_text_for_node(
656
- title=row["title"],
657
- summary=row["summary"] or "",
658
- metadata=metadata,
659
- )
660
- if text:
661
- items.append(
662
- {
663
- "item_id": row["id"],
664
- "item_type": "node",
665
- "source_node": row["id"],
666
- "text": text,
667
- "metadata": {"node_type": row["type"], **metadata},
668
- }
669
- )
670
- if include_chunks:
671
- for row in conn.execute(
672
- """
673
- SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
674
- FROM chunks c
675
- JOIN nodes n ON n.id=c.id
676
- ORDER BY c.created_at DESC, c.id ASC
677
- """
678
- ).fetchall():
679
- metadata = _safe_loads(row["metadata_json"])
680
- text = _clean_text(row["text"] or "")
681
- if text:
682
- items.append(
683
- {
684
- "item_id": row["id"],
685
- "item_type": "chunk",
686
- "source_node": row["id"],
687
- "text": text,
688
- "metadata": {
689
- **metadata,
690
- "parent_source_node": row["parent_source_node"],
691
- },
692
- }
693
- )
694
- return items
695
-
696
- def rebuild_vector_index(
697
- self,
698
- *,
699
- full: bool = False,
700
- include_nodes: bool = True,
701
- include_chunks: bool = True,
702
- ) -> Dict[str, Any]:
703
- """Rebuild the derived vector index without mutating graph content."""
704
- op_id = f"vector-op:{_sha256_text(f'{time.time()}:{os.getpid()}')[:24]}"
705
- requested_at = _now()
706
- started = time.perf_counter()
707
- try:
708
- with self._connect() as conn:
709
- conn.execute(
710
- """
711
- INSERT INTO vector_index_operations(
712
- id, operation, status, requested_at, started_at, metadata_json
713
- )
714
- VALUES (?, ?, 'running', ?, ?, ?)
715
- """,
716
- (
717
- op_id,
718
- "rebuild_full" if full else "rebuild_incremental",
719
- requested_at,
720
- requested_at,
721
- _json(
722
- {
723
- "include_nodes": include_nodes,
724
- "include_chunks": include_chunks,
725
- }
726
- ),
727
- ),
728
- )
729
- if full:
730
- filters = []
731
- if include_nodes:
732
- filters.append("'node'")
733
- if include_chunks:
734
- filters.append("'chunk'")
735
- if filters:
736
- conn.execute(
737
- f"DELETE FROM vector_embeddings WHERE item_type IN ({','.join(filters)})"
738
- )
739
- items = self._iter_vector_source_items(
740
- conn,
741
- include_nodes=include_nodes,
742
- include_chunks=include_chunks,
743
- )
744
- indexed = skipped = 0
745
- for item in items:
746
- changed = self._upsert_vector_item(conn, **item)
747
- if changed:
748
- indexed += 1
749
- else:
750
- skipped += 1
751
- duration_ms = round((time.perf_counter() - started) * 1000, 2)
752
- conn.execute(
753
- """
754
- UPDATE vector_index_operations
755
- SET status='completed', completed_at=?, items_total=?,
756
- items_indexed=?, items_skipped=?, metadata_json=?
757
- WHERE id=?
758
- """,
759
- (
760
- _now(),
761
- len(items),
762
- indexed,
763
- skipped,
764
- _json(
765
- {
766
- "include_nodes": include_nodes,
767
- "include_chunks": include_chunks,
768
- "duration_ms": duration_ms,
769
- "embedding_model": self._embedding_model.model_id,
770
- "embedding_dim": self._embedding_model.dim,
771
- }
772
- ),
773
- op_id,
774
- ),
775
- )
776
- return {
777
- "status": "completed",
778
- "operation_id": op_id,
779
- "full": bool(full),
780
- "items_total": len(items),
781
- "items_indexed": indexed,
782
- "items_skipped": skipped,
783
- "duration_ms": duration_ms,
784
- "embedding_model": self._embedding_model.model_id,
785
- "embedding_dim": self._embedding_model.dim,
786
- }
787
- except Exception as exc:
788
- duration_ms = round((time.perf_counter() - started) * 1000, 2)
789
- with self._connect() as conn:
790
- conn.execute(
791
- """
792
- INSERT INTO vector_index_operations(
793
- id, operation, status, requested_at, started_at, completed_at,
794
- error_message, metadata_json
795
- )
796
- VALUES (?, ?, 'failed', ?, ?, ?, ?, ?)
797
- ON CONFLICT(id) DO UPDATE SET
798
- status='failed',
799
- completed_at=excluded.completed_at,
800
- error_message=excluded.error_message,
801
- metadata_json=excluded.metadata_json
802
- """,
803
- (
804
- op_id,
805
- "rebuild_full" if full else "rebuild_incremental",
806
- requested_at,
807
- requested_at,
808
- _now(),
809
- str(exc),
810
- _json({"duration_ms": duration_ms}),
811
- ),
812
- )
813
- raise
814
-
815
- def index_status(self) -> Dict[str, Any]:
816
- storage_capabilities = None
817
- try:
818
- storage_capabilities = self.storage_engine.capabilities().as_dict()
819
- except Exception as exc:
820
- storage_capabilities = {
821
- "engine": "sqlite",
822
- "available": False,
823
- "reason": str(exc),
824
- }
825
- with self._connect() as conn:
826
- vector_counts = {
827
- row["item_type"]: row["count"]
828
- for row in conn.execute(
829
- "SELECT item_type, COUNT(*) AS count FROM vector_embeddings GROUP BY item_type"
830
- )
831
- }
832
- source_items = self._iter_vector_source_items(conn)
833
- vector_rows = {
834
- row["item_id"]: row
835
- for row in conn.execute(
836
- """
837
- SELECT item_id, text_hash, embedding_dim, embedding_model, indexed_at
838
- FROM vector_embeddings
839
- """
840
- ).fetchall()
841
- }
842
- latest_rows = conn.execute(
843
- """
844
- SELECT id, operation, status, requested_at, started_at, completed_at,
845
- items_total, items_indexed, items_skipped, error_message, metadata_json
846
- FROM vector_index_operations
847
- ORDER BY requested_at DESC, id DESC
848
- LIMIT 5
849
- """
850
- ).fetchall()
851
- missing = stale = ready = 0
852
- for item in source_items:
853
- vector_row = vector_rows.get(item["item_id"])
854
- expected_hash = _sha256_text(_clean_text(item["text"]))
855
- if not vector_row:
856
- missing += 1
857
- elif (
858
- vector_row["text_hash"] != expected_hash
859
- or vector_row["embedding_dim"] != self._embedding_model.dim
860
- or vector_row["embedding_model"] != self._embedding_model.model_id
861
- ):
862
- stale += 1
863
- else:
864
- ready += 1
865
- pending = missing + stale
866
- return {
867
- "status": "ready" if pending == 0 else "needs_reindex",
868
- "storage": {
869
- "db_path": str(self.db_path),
870
- "backend": "sqlite",
871
- "embedding_model": self._embedding_model.model_id,
872
- "embedding_dim": self._embedding_model.dim,
873
- # Honest capability report: trigram FTS5 keyword index, or
874
- # LIKE-scan fallback when this SQLite build lacks it.
875
- "fts_enabled": bool(getattr(self, "_fts_enabled", False)),
876
- "engine": storage_capabilities,
877
- "vector_search_backend": (
878
- storage_capabilities.get("vector_backend")
879
- if isinstance(storage_capabilities, dict)
880
- else "bruteforce-cosine"
881
- ),
882
- "vector_search_mode": (
883
- (storage_capabilities.get("metadata") or {}).get("vector_mode")
884
- if isinstance(storage_capabilities, dict)
885
- else "fallback"
886
- ),
887
- "sqlite_vec_ann_available": (
888
- bool((storage_capabilities.get("metadata") or {}).get("sqlite_vec_ann_available"))
889
- if isinstance(storage_capabilities, dict)
890
- else False
891
- ),
892
- },
893
- "source_items": len(source_items),
894
- "indexed_items": sum(vector_counts.values()),
895
- "ready_items": ready,
896
- "missing_items": missing,
897
- "stale_items": stale,
898
- "pending_items": pending,
899
- "by_item_type": vector_counts,
900
- "operations": [
901
- {
902
- "id": row["id"],
903
- "operation": row["operation"],
904
- "status": row["status"],
905
- "requested_at": row["requested_at"],
906
- "started_at": row["started_at"],
907
- "completed_at": row["completed_at"],
908
- "items_total": row["items_total"],
909
- "items_indexed": row["items_indexed"],
910
- "items_skipped": row["items_skipped"],
911
- "error_message": row["error_message"],
912
- "metadata": _safe_loads(row["metadata_json"]),
913
- }
914
- for row in latest_rows
915
- ],
916
- }
917
-
918
- def vector_search(
919
- self,
920
- query: str,
921
- *,
922
- limit: int = 30,
923
- min_score: float = 0.0,
924
- max_candidates: int = 10_000,
925
- ) -> Dict[str, Any]:
926
- query = str(query or "").strip()
927
- limit = max(1, min(int(limit or 30), 100))
928
- min_score = float(min_score or 0.0)
929
- if not query:
930
- return {"query": query, "matches": []}
931
- query_vector = self._embedding_model.embed(query)
932
- max_candidates = max(limit, min(int(max_candidates or 10_000), 50_000))
933
- with self._connect() as conn:
934
- rows = conn.execute(
935
- """
936
- SELECT
937
- ve.item_id, ve.item_type, ve.source_node, ve.embedding,
938
- ve.embedding_dim, ve.embedding_model, ve.metadata_json AS vector_metadata,
939
- n.type AS node_type, n.title AS node_title, n.summary AS node_summary,
940
- n.metadata_json AS node_metadata, n.updated_at AS node_updated_at,
941
- c.text AS chunk_text, c.source_node AS parent_node_id,
942
- pn.type AS parent_type, pn.title AS parent_title,
943
- pn.summary AS parent_summary, pn.metadata_json AS parent_metadata,
944
- pn.updated_at AS parent_updated_at
945
- FROM vector_embeddings ve
946
- LEFT JOIN nodes n ON n.id=ve.source_node
947
- LEFT JOIN chunks c ON c.id=ve.item_id
948
- LEFT JOIN nodes pn ON pn.id=c.source_node
949
- WHERE ve.embedding_model=? AND ve.embedding_dim=?
950
- ORDER BY ve.indexed_at DESC
951
- LIMIT ?
952
- """,
953
- (
954
- self._embedding_model.model_id,
955
- self._embedding_model.dim,
956
- max_candidates,
957
- ),
958
- ).fetchall()
959
- scored = []
960
- for row in rows:
961
- vector = self._embedding_model.decode(
962
- row["embedding"], row["embedding_dim"]
963
- )
964
- score = self._embedding_model.similarity(query_vector, vector)
965
- if score < min_score:
966
- continue
967
- is_chunk = row["item_type"] == "chunk"
968
- summary = (
969
- row["chunk_text"]
970
- if is_chunk and row["chunk_text"]
971
- else row["node_summary"]
972
- )
973
- parent_metadata = _safe_loads(row["parent_metadata"])
974
- node_metadata = _safe_loads(row["node_metadata"])
975
- scored.append(
976
- {
977
- "id": row["item_id"],
978
- "node_id": row["parent_node_id"]
979
- if is_chunk and row["parent_node_id"]
980
- else row["source_node"],
981
- "item_type": row["item_type"],
982
- "type": "Chunk" if is_chunk else row["node_type"],
983
- "title": row["parent_title"]
984
- if is_chunk and row["parent_title"]
985
- else row["node_title"],
986
- "summary": _clean_text(summary or "")[:1000],
987
- "score": round(float(score), 6),
988
- "metadata": {
989
- **(parent_metadata if is_chunk else node_metadata),
990
- "vector": _safe_loads(row["vector_metadata"]),
991
- "parent_node_id": row["parent_node_id"],
992
- "parent_type": row["parent_type"],
993
- },
994
- "updated_at": row["parent_updated_at"]
995
- if is_chunk and row["parent_updated_at"]
996
- else row["node_updated_at"],
997
- }
998
- )
999
- scored.sort(
1000
- key=lambda item: (item["score"], item.get("updated_at") or ""), reverse=True
1001
- )
1002
- return {
1003
- "query": query,
1004
- "embedding_model": self._embedding_model.model_id,
1005
- "embedding_dim": self._embedding_model.dim,
1006
- "matches": scored[:limit],
1007
- }
668
+ ]
669
+ edges = list(edges_by_id.values())
670
+ if allowed_workspaces is not None:
671
+ nodes = self.filter_scoped_nodes(nodes, allowed_workspaces)
672
+ kept = {node.get("id") for node in nodes}
673
+ edges = [edge for edge in edges if edge.get("from") in kept and edge.get("to") in kept]
674
+ return {"root": node_id, "depth": depth, "nodes": nodes, "edges": edges}
1008
675
 
1009
676
  def delete_conversation(self, conversation_id: str) -> Dict[str, Any]:
1010
677
  conversation_id = str(conversation_id or "").strip()
@@ -1141,201 +808,3 @@ class KnowledgeGraphRetrievalMixin:
1141
808
  "local_file_status": local_file_status,
1142
809
  "v2": v2,
1143
810
  }
1144
-
1145
- def search_for_document_generation(
1146
- self, query: str, limit: int = 10
1147
- ) -> List[Dict[str, Any]]:
1148
- """Hybrid retrieval optimized for document generation.
1149
-
1150
- Scoring: 0.5*text_relevance + 0.3*graph_relationship + 0.2*recency
1151
- Returns nodes with rich context for document generation prompts.
1152
- """
1153
- query = str(query or "").strip()
1154
- if not query:
1155
- return []
1156
- limit = max(1, min(int(limit or 10), 50))
1157
- terms = _topic_candidates(query, limit=12)
1158
- now = datetime.now()
1159
- nt, et = self._read_tables()
1160
-
1161
- with self._connect() as conn:
1162
- candidate_rows = []
1163
- seen_ids = set()
1164
-
1165
- if query:
1166
- q = f"%{query}%"
1167
- rows = conn.execute(
1168
- f"""
1169
- SELECT id, type, title, summary, metadata_json, updated_at
1170
- FROM {nt}
1171
- WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
1172
- AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
1173
- 'Spreadsheet', 'Image', 'ImageText', 'Chat',
1174
- 'Decision', 'Task', 'Concept', 'Feature',
1175
- 'Page', 'Slide')
1176
- ORDER BY updated_at DESC, id ASC
1177
- LIMIT ?
1178
- """,
1179
- (q, q, q, limit * 5),
1180
- ).fetchall()
1181
- for row in rows:
1182
- if row["id"] not in seen_ids:
1183
- seen_ids.add(row["id"])
1184
- candidate_rows.append(row)
1185
-
1186
- for term in terms:
1187
- t = f"%{term}%"
1188
- rows = conn.execute(
1189
- f"""
1190
- SELECT id, type, title, summary, metadata_json, updated_at
1191
- FROM {nt}
1192
- WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
1193
- AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
1194
- 'Spreadsheet', 'Image', 'ImageText', 'Chat',
1195
- 'Decision', 'Task', 'Concept', 'Feature',
1196
- 'Page', 'Slide')
1197
- ORDER BY updated_at DESC, id ASC
1198
- LIMIT ?
1199
- """,
1200
- (t, t, t, limit * 3),
1201
- ).fetchall()
1202
- for row in rows:
1203
- if row["id"] not in seen_ids:
1204
- seen_ids.add(row["id"])
1205
- candidate_rows.append(row)
1206
-
1207
- scored_results = []
1208
- for row in candidate_rows:
1209
- haystack = (
1210
- f"{row['title']} {row['summary']} {row['metadata_json']}".lower()
1211
- )
1212
-
1213
- text_hits = sum(1 for term in terms if term.lower() in haystack)
1214
- text_score = min(1.0, text_hits / max(len(terms), 1))
1215
-
1216
- edge_count = conn.execute(
1217
- f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
1218
- (row["id"], row["id"]),
1219
- ).fetchone()["c"]
1220
- graph_score = min(1.0, math.log1p(edge_count) / 4.0)
1221
-
1222
- recency = _recency_score(
1223
- row["updated_at"], now=now, half_life_days=14.0
1224
- )
1225
-
1226
- doc_type_boost = (
1227
- 1.2
1228
- if row["type"]
1229
- in (
1230
- "Document",
1231
- "File",
1232
- "SlideDeck",
1233
- "Decision",
1234
- )
1235
- else 1.0
1236
- )
1237
-
1238
- hybrid_score = (
1239
- 0.5 * text_score + 0.3 * graph_score + 0.2 * recency
1240
- ) * doc_type_boost
1241
-
1242
- meta = _safe_loads(row["metadata_json"])
1243
- neighbor_concepts = []
1244
- neighbor_rows = conn.execute(
1245
- f"""
1246
- SELECT n.title, n.type FROM {et} e
1247
- JOIN {nt} n ON n.id = CASE WHEN e.from_node = ? THEN e.to_node ELSE e.from_node END
1248
- WHERE (e.from_node = ? OR e.to_node = ?)
1249
- AND n.type IN ('Concept', 'Feature', 'Decision', 'Task')
1250
- LIMIT 8
1251
- """,
1252
- (row["id"], row["id"], row["id"]),
1253
- ).fetchall()
1254
- for nr in neighbor_rows:
1255
- neighbor_concepts.append({"title": nr["title"], "type": nr["type"]})
1256
-
1257
- scored_results.append(
1258
- {
1259
- "id": row["id"],
1260
- "type": row["type"],
1261
- "title": row["title"],
1262
- "summary": row["summary"],
1263
- "metadata": meta,
1264
- "updated_at": row["updated_at"],
1265
- "hybrid_score": round(hybrid_score, 4),
1266
- "scores": {
1267
- "text": round(text_score, 4),
1268
- "graph": round(graph_score, 4),
1269
- "recency": round(recency, 4),
1270
- },
1271
- "related_concepts": neighbor_concepts,
1272
- }
1273
- )
1274
-
1275
- scored_results.sort(key=lambda x: x["hybrid_score"], reverse=True)
1276
- return scored_results[:limit]
1277
-
1278
- def multi_hop_context(
1279
- self, node_ids: List[str], max_hops: int = 2
1280
- ) -> Dict[str, Any]:
1281
- """Multi-hop graph traversal from seed nodes for richer context."""
1282
- visited_nodes = set()
1283
- visited_edges = set()
1284
- all_nodes = []
1285
- all_edges = []
1286
- frontier = set(node_ids)
1287
- nt, et = self._read_tables()
1288
-
1289
- with self._connect() as conn:
1290
- for hop in range(max_hops):
1291
- if not frontier:
1292
- break
1293
- next_frontier = set()
1294
- for nid in frontier:
1295
- if nid in visited_nodes:
1296
- continue
1297
- visited_nodes.add(nid)
1298
- row = conn.execute(
1299
- f"SELECT id, type, title, summary, metadata_json, updated_at FROM {nt} WHERE id=?",
1300
- (nid,),
1301
- ).fetchone()
1302
- if row:
1303
- all_nodes.append(
1304
- {
1305
- "id": row["id"],
1306
- "type": row["type"],
1307
- "title": row["title"],
1308
- "summary": row["summary"],
1309
- "metadata": _safe_loads(row["metadata_json"]),
1310
- "hop": hop,
1311
- }
1312
- )
1313
- edge_rows = conn.execute(
1314
- f"""
1315
- SELECT id, from_node, to_node, type, weight
1316
- FROM {et} WHERE from_node=? OR to_node=?
1317
- ORDER BY id ASC
1318
- """,
1319
- (nid, nid),
1320
- ).fetchall()
1321
- for er in edge_rows:
1322
- if er["id"] not in visited_edges:
1323
- visited_edges.add(er["id"])
1324
- all_edges.append(
1325
- {
1326
- "from": er["from_node"],
1327
- "to": er["to_node"],
1328
- "type": er["type"],
1329
- "weight": er["weight"],
1330
- }
1331
- )
1332
- other = (
1333
- er["to_node"]
1334
- if er["from_node"] == nid
1335
- else er["from_node"]
1336
- )
1337
- if other not in visited_nodes:
1338
- next_frontier.add(other)
1339
- frontier = next_frontier
1340
-
1341
- return {"nodes": all_nodes, "edges": all_edges}