ltcai 8.8.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -37
- package/auto_setup.py +84 -70
- package/docs/CHANGELOG.md +113 -237
- package/docs/CODE_REVIEW_2026-07-06.md +764 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +3 -3
- package/docs/DEVELOPMENT.md +10 -10
- package/docs/LEGACY_COMPATIBILITY.md +1 -1
- package/docs/ONBOARDING.md +2 -2
- package/docs/PRODUCT_DIRECTION_REVIEW.md +3 -2
- package/docs/ROADMAP_RECOMMENDATIONS.md +61 -36
- package/docs/TRUST_MODEL.md +5 -1
- package/docs/WHY_LATTICE.md +4 -3
- package/docs/architecture.md +4 -0
- package/docs/kg-schema.md +1 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/archive.py +4 -9
- package/lattice_brain/conversations.py +156 -21
- package/lattice_brain/embeddings.py +38 -2
- package/lattice_brain/graph/_kg_common.py +39 -496
- package/lattice_brain/graph/_kg_constants.py +243 -0
- package/lattice_brain/graph/_kg_fsutil.py +297 -0
- package/lattice_brain/graph/discovery.py +0 -948
- package/lattice_brain/graph/discovery_index.py +972 -0
- package/lattice_brain/graph/json_utils.py +25 -0
- package/lattice_brain/graph/retrieval.py +66 -597
- package/lattice_brain/graph/retrieval_docgen.py +210 -0
- package/lattice_brain/graph/retrieval_vector.py +460 -0
- package/lattice_brain/graph/runtime.py +16 -0
- package/lattice_brain/graph/store.py +6 -0
- package/lattice_brain/ingestion.py +68 -0
- package/lattice_brain/portability.py +1 -9
- package/lattice_brain/quality.py +98 -4
- package/lattice_brain/runtime/agent_runtime.py +166 -0
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/lattice_brain/utils.py +28 -0
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/chat.py +368 -418
- package/latticeai/api/chat_helpers.py +227 -0
- package/latticeai/api/computer_use.py +149 -31
- package/latticeai/api/marketplace.py +11 -0
- package/latticeai/api/mcp.py +3 -2
- package/latticeai/api/models.py +4 -1
- package/latticeai/api/permissions.py +72 -33
- package/latticeai/api/setup.py +17 -2
- package/latticeai/api/tools.py +105 -62
- package/latticeai/app_factory.py +101 -296
- package/latticeai/core/agent.py +25 -7
- package/latticeai/core/io_utils.py +37 -0
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/local_embeddings.py +2 -4
- package/latticeai/core/marketplace.py +33 -2
- package/latticeai/core/mcp_catalog.py +450 -0
- package/latticeai/core/mcp_registry.py +2 -441
- package/latticeai/core/sessions.py +11 -3
- package/latticeai/core/tool_registry.py +15 -4
- package/latticeai/core/users.py +4 -9
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/core/workspace_os_utils.py +3 -17
- package/latticeai/integrations/telegram_bot.py +7 -2
- package/latticeai/models/model_providers.py +111 -0
- package/latticeai/models/router.py +58 -136
- package/latticeai/runtime/audit_runtime.py +27 -16
- package/latticeai/runtime/automation_runtime.py +9 -0
- package/latticeai/runtime/bootstrap.py +1 -1
- package/latticeai/runtime/history_runtime.py +163 -0
- package/latticeai/runtime/namespace_runtime.py +173 -0
- package/latticeai/runtime/network_config_runtime.py +56 -0
- package/latticeai/runtime/sso_config_runtime.py +128 -0
- package/latticeai/runtime/user_key_runtime.py +106 -0
- package/latticeai/services/app_context.py +1 -0
- package/latticeai/services/architecture_readiness.py +2 -2
- package/latticeai/services/memory_service.py +213 -0
- package/latticeai/services/model_engines.py +79 -12
- package/latticeai/services/model_runtime.py +24 -4
- package/latticeai/services/platform_runtime.py +9 -1
- package/latticeai/services/process_audit.py +208 -0
- package/latticeai/services/product_readiness.py +11 -11
- package/latticeai/services/review_queue.py +64 -11
- package/latticeai/services/run_executor.py +21 -0
- package/latticeai/services/search_service.py +106 -30
- package/latticeai/services/setup_detection.py +80 -0
- package/latticeai/services/tool_dispatch.py +66 -0
- package/latticeai/services/workspace_service.py +15 -0
- package/package.json +1 -1
- package/scripts/check_i18n_literals.mjs +20 -8
- package/scripts/i18n_literal_allowlist.json +34 -0
- package/scripts/lint_frontend.mjs +6 -2
- package/setup_wizard.py +196 -74
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +11 -11
- package/static/app/assets/{Act-C7K9wsO9.js → Act-21lIXx2E.js} +1 -1
- package/static/app/assets/Brain-BqUd5UJJ.js +321 -0
- package/static/app/assets/{Capture-B3V4_5Xp.js → Capture-BA7Z2Q1u.js} +1 -1
- package/static/app/assets/{Library-Cgj-EF50.js → Library-bFMtyni3.js} +1 -1
- package/static/app/assets/System-K6krGCqn.js +1 -0
- package/static/app/assets/index-C4R3ws30.js +17 -0
- package/static/app/assets/index-ChSeOB02.css +2 -0
- package/static/app/assets/primitives-sQU3it5I.js +1 -0
- package/static/app/assets/{textarea-CVQkN2Tk.js → textarea-DK3Fd_lR.js} +1 -1
- package/static/app/index.html +2 -2
- package/static/css/tokens.css +4 -2
- package/static/sw.js +1 -1
- package/tools/local_files.py +6 -0
- package/latticeai/runtime/sso_runtime.py +0 -52
- package/static/app/assets/Brain-I1OSzxJu.js +0 -321
- package/static/app/assets/System-D1Lkei3I.js +0 -1
- package/static/app/assets/index--P0ksosz.js +0 -17
- package/static/app/assets/index-DCh5AoXt.css +0 -2
- package/static/app/assets/primitives-BLqaKk5g.js +0 -1
|
@@ -55,9 +55,18 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
55
55
|
extracted = meta.get("extracted") or {}
|
|
56
56
|
node_id = row["id"]
|
|
57
57
|
chunk_count = conn.execute(
|
|
58
|
-
|
|
59
|
-
(
|
|
58
|
+
"SELECT COUNT(*) AS c FROM chunks WHERE source_node=?",
|
|
59
|
+
(node_id,),
|
|
60
60
|
).fetchone()["c"]
|
|
61
|
+
if not chunk_count:
|
|
62
|
+
# Legacy projections represented chunks as graph nodes and
|
|
63
|
+
# linked them only through metadata_json. Keep read
|
|
64
|
+
# compatibility without making the fragile LIKE path the
|
|
65
|
+
# primary query.
|
|
66
|
+
chunk_count = conn.execute(
|
|
67
|
+
f"SELECT COUNT(*) AS c FROM {nt} WHERE type='Chunk' AND metadata_json LIKE ?",
|
|
68
|
+
(f"%{node_id}%",),
|
|
69
|
+
).fetchone()["c"]
|
|
61
70
|
documents.append(
|
|
62
71
|
{
|
|
63
72
|
"id": node_id,
|
|
@@ -255,7 +264,7 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
255
264
|
node.pop("_raw_importance", None)
|
|
256
265
|
return {"nodes": nodes, "edges": edges}
|
|
257
266
|
|
|
258
|
-
def search(self, query: str, limit: int = 30) -> Dict[str, Any]:
|
|
267
|
+
def search(self, query: str, limit: int = 30, *, allowed_workspaces=None) -> Dict[str, Any]:
|
|
259
268
|
query = str(query or "").strip()
|
|
260
269
|
q = f"%{query}%"
|
|
261
270
|
limit = max(1, min(int(limit or 30), 100))
|
|
@@ -347,9 +356,7 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
347
356
|
# the legacy LIKE path regardless of FTS bm25 tie ordering.
|
|
348
357
|
rows = sorted(rows, key=lambda r: r["id"])
|
|
349
358
|
rows = sorted(rows, key=score, reverse=True)[:limit]
|
|
350
|
-
|
|
351
|
-
"query": query,
|
|
352
|
-
"matches": [
|
|
359
|
+
matches = [
|
|
353
360
|
{
|
|
354
361
|
"id": row["id"],
|
|
355
362
|
"type": row["type"],
|
|
@@ -359,15 +366,17 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
359
366
|
"updated_at": row["updated_at"],
|
|
360
367
|
}
|
|
361
368
|
for row in rows
|
|
362
|
-
]
|
|
363
|
-
|
|
369
|
+
]
|
|
370
|
+
if allowed_workspaces is not None:
|
|
371
|
+
matches = self.filter_scoped_nodes(matches, allowed_workspaces)
|
|
372
|
+
return {"query": query, "matches": matches}
|
|
364
373
|
|
|
365
|
-
def context_for_query(self, query: str, limit: int = 6) -> str:
|
|
374
|
+
def context_for_query(self, query: str, limit: int = 6, *, allowed_workspaces=None) -> str:
|
|
366
375
|
"""Return compact graph-backed RAG context for chat generation."""
|
|
367
376
|
query = str(query or "").strip()
|
|
368
377
|
if not query:
|
|
369
378
|
return ""
|
|
370
|
-
matches = self.search(query, limit).get("matches", [])
|
|
379
|
+
matches = self.search(query, limit, allowed_workspaces=allowed_workspaces).get("matches", [])
|
|
371
380
|
if not matches:
|
|
372
381
|
topics = _topic_candidates(query, limit=4)
|
|
373
382
|
if topics:
|
|
@@ -404,6 +413,8 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
404
413
|
)
|
|
405
414
|
if len(matches) >= limit:
|
|
406
415
|
break
|
|
416
|
+
if allowed_workspaces is not None:
|
|
417
|
+
matches = self.filter_scoped_nodes(matches, allowed_workspaces)
|
|
407
418
|
lines = []
|
|
408
419
|
for match in matches[:limit]:
|
|
409
420
|
meta = match.get("metadata") or {}
|
|
@@ -420,8 +431,10 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
420
431
|
)
|
|
421
432
|
return "\n".join(lines)
|
|
422
433
|
|
|
423
|
-
def neighbors(self, node_id: str) -> Dict[str, Any]:
|
|
434
|
+
def neighbors(self, node_id: str, *, allowed_workspaces=None) -> Dict[str, Any]:
|
|
424
435
|
"""Return direct neighbors (1-hop) of a node."""
|
|
436
|
+
if allowed_workspaces is not None and not self.filter_scoped_nodes([{"id": node_id}], allowed_workspaces):
|
|
437
|
+
raise ValueError(f"graph node not found: {node_id}")
|
|
425
438
|
nt, et = self._read_tables()
|
|
426
439
|
with self._connect() as conn:
|
|
427
440
|
edge_rows = conn.execute(
|
|
@@ -458,9 +471,17 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
458
471
|
list(neighbor_ids),
|
|
459
472
|
)
|
|
460
473
|
]
|
|
474
|
+
if allowed_workspaces is not None:
|
|
475
|
+
nodes = self.filter_scoped_nodes(nodes, allowed_workspaces)
|
|
476
|
+
kept = {node.get("id") for node in nodes}
|
|
477
|
+
edges = [
|
|
478
|
+
edge for edge in edges
|
|
479
|
+
if (edge.get("from") == node_id or edge.get("from") in kept)
|
|
480
|
+
and (edge.get("to") == node_id or edge.get("to") in kept)
|
|
481
|
+
]
|
|
461
482
|
return {"node_id": node_id, "neighbors": nodes, "edges": edges}
|
|
462
483
|
|
|
463
|
-
def get_node(self, node_id: str) -> Dict[str, Any]:
|
|
484
|
+
def get_node(self, node_id: str, *, allowed_workspaces=None) -> Dict[str, Any]:
|
|
464
485
|
node_id = str(node_id or "").strip()
|
|
465
486
|
if not node_id:
|
|
466
487
|
raise ValueError("node_id required")
|
|
@@ -480,7 +501,7 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
480
501
|
f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
|
|
481
502
|
(node_id, node_id),
|
|
482
503
|
).fetchone()["c"]
|
|
483
|
-
|
|
504
|
+
node = {
|
|
484
505
|
"id": row["id"],
|
|
485
506
|
"type": row["type"],
|
|
486
507
|
"title": row["title"],
|
|
@@ -489,6 +510,9 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
489
510
|
"updated_at": row["updated_at"],
|
|
490
511
|
"degree": degree,
|
|
491
512
|
}
|
|
513
|
+
if allowed_workspaces is not None and not self.filter_scoped_nodes([node], allowed_workspaces):
|
|
514
|
+
raise ValueError(f"graph node not found: {node_id}")
|
|
515
|
+
return node
|
|
492
516
|
|
|
493
517
|
def relationship_search(
|
|
494
518
|
self,
|
|
@@ -497,6 +521,7 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
497
521
|
node_id: str = "",
|
|
498
522
|
relationship_type: str = "",
|
|
499
523
|
limit: int = 30,
|
|
524
|
+
allowed_workspaces=None,
|
|
500
525
|
) -> Dict[str, Any]:
|
|
501
526
|
query = str(query or "").strip()
|
|
502
527
|
node_id = str(node_id or "").strip()
|
|
@@ -535,11 +560,7 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
535
560
|
""",
|
|
536
561
|
(*params, limit),
|
|
537
562
|
).fetchall()
|
|
538
|
-
|
|
539
|
-
"query": query,
|
|
540
|
-
"node_id": node_id,
|
|
541
|
-
"relationship_type": relationship_type,
|
|
542
|
-
"relationships": [
|
|
563
|
+
relationships = [
|
|
543
564
|
{
|
|
544
565
|
"id": row["id"],
|
|
545
566
|
"type": row["type"],
|
|
@@ -562,15 +583,32 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
562
583
|
},
|
|
563
584
|
}
|
|
564
585
|
for row in rows
|
|
565
|
-
]
|
|
586
|
+
]
|
|
587
|
+
if allowed_workspaces is not None:
|
|
588
|
+
kept = []
|
|
589
|
+
for rel in relationships:
|
|
590
|
+
endpoints = [
|
|
591
|
+
{"id": (rel.get("source") or {}).get("id")},
|
|
592
|
+
{"id": (rel.get("target") or {}).get("id")},
|
|
593
|
+
]
|
|
594
|
+
if len(self.filter_scoped_nodes(endpoints, allowed_workspaces)) == 2:
|
|
595
|
+
kept.append(rel)
|
|
596
|
+
relationships = kept
|
|
597
|
+
return {
|
|
598
|
+
"query": query,
|
|
599
|
+
"node_id": node_id,
|
|
600
|
+
"relationship_type": relationship_type,
|
|
601
|
+
"relationships": relationships,
|
|
566
602
|
}
|
|
567
603
|
|
|
568
604
|
def traverse(
|
|
569
|
-
self, node_id: str, *, depth: int = 1, limit: int = 100
|
|
605
|
+
self, node_id: str, *, depth: int = 1, limit: int = 100, allowed_workspaces=None
|
|
570
606
|
) -> Dict[str, Any]:
|
|
571
607
|
node_id = str(node_id or "").strip()
|
|
572
608
|
if not node_id:
|
|
573
609
|
raise ValueError("node_id required")
|
|
610
|
+
if allowed_workspaces is not None and not self.filter_scoped_nodes([{"id": node_id}], allowed_workspaces):
|
|
611
|
+
raise ValueError(f"graph node not found: {node_id}")
|
|
574
612
|
depth = max(0, min(int(depth or 1), 4))
|
|
575
613
|
limit = max(1, min(int(limit or 100), 500))
|
|
576
614
|
nt, et = self._read_tables()
|
|
@@ -617,10 +655,7 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
617
655
|
""",
|
|
618
656
|
list(visited),
|
|
619
657
|
).fetchall()
|
|
620
|
-
|
|
621
|
-
"root": node_id,
|
|
622
|
-
"depth": depth,
|
|
623
|
-
"nodes": [
|
|
658
|
+
nodes = [
|
|
624
659
|
{
|
|
625
660
|
"id": row["id"],
|
|
626
661
|
"type": row["type"],
|
|
@@ -630,381 +665,13 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
630
665
|
"updated_at": row["updated_at"],
|
|
631
666
|
}
|
|
632
667
|
for row in node_rows
|
|
633
|
-
]
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
*,
|
|
641
|
-
include_nodes: bool = True,
|
|
642
|
-
include_chunks: bool = True,
|
|
643
|
-
) -> List[Dict[str, Any]]:
|
|
644
|
-
items: List[Dict[str, Any]] = []
|
|
645
|
-
if include_nodes:
|
|
646
|
-
for row in conn.execute(
|
|
647
|
-
"""
|
|
648
|
-
SELECT id, type, title, summary, metadata_json
|
|
649
|
-
FROM nodes
|
|
650
|
-
WHERE type <> 'Chunk'
|
|
651
|
-
ORDER BY updated_at DESC, id ASC
|
|
652
|
-
"""
|
|
653
|
-
).fetchall():
|
|
654
|
-
metadata = _safe_loads(row["metadata_json"])
|
|
655
|
-
text = self._vector_text_for_node(
|
|
656
|
-
title=row["title"],
|
|
657
|
-
summary=row["summary"] or "",
|
|
658
|
-
metadata=metadata,
|
|
659
|
-
)
|
|
660
|
-
if text:
|
|
661
|
-
items.append(
|
|
662
|
-
{
|
|
663
|
-
"item_id": row["id"],
|
|
664
|
-
"item_type": "node",
|
|
665
|
-
"source_node": row["id"],
|
|
666
|
-
"text": text,
|
|
667
|
-
"metadata": {"node_type": row["type"], **metadata},
|
|
668
|
-
}
|
|
669
|
-
)
|
|
670
|
-
if include_chunks:
|
|
671
|
-
for row in conn.execute(
|
|
672
|
-
"""
|
|
673
|
-
SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
|
|
674
|
-
FROM chunks c
|
|
675
|
-
JOIN nodes n ON n.id=c.id
|
|
676
|
-
ORDER BY c.created_at DESC, c.id ASC
|
|
677
|
-
"""
|
|
678
|
-
).fetchall():
|
|
679
|
-
metadata = _safe_loads(row["metadata_json"])
|
|
680
|
-
text = _clean_text(row["text"] or "")
|
|
681
|
-
if text:
|
|
682
|
-
items.append(
|
|
683
|
-
{
|
|
684
|
-
"item_id": row["id"],
|
|
685
|
-
"item_type": "chunk",
|
|
686
|
-
"source_node": row["id"],
|
|
687
|
-
"text": text,
|
|
688
|
-
"metadata": {
|
|
689
|
-
**metadata,
|
|
690
|
-
"parent_source_node": row["parent_source_node"],
|
|
691
|
-
},
|
|
692
|
-
}
|
|
693
|
-
)
|
|
694
|
-
return items
|
|
695
|
-
|
|
696
|
-
def rebuild_vector_index(
|
|
697
|
-
self,
|
|
698
|
-
*,
|
|
699
|
-
full: bool = False,
|
|
700
|
-
include_nodes: bool = True,
|
|
701
|
-
include_chunks: bool = True,
|
|
702
|
-
) -> Dict[str, Any]:
|
|
703
|
-
"""Rebuild the derived vector index without mutating graph content."""
|
|
704
|
-
op_id = f"vector-op:{_sha256_text(f'{time.time()}:{os.getpid()}')[:24]}"
|
|
705
|
-
requested_at = _now()
|
|
706
|
-
started = time.perf_counter()
|
|
707
|
-
try:
|
|
708
|
-
with self._connect() as conn:
|
|
709
|
-
conn.execute(
|
|
710
|
-
"""
|
|
711
|
-
INSERT INTO vector_index_operations(
|
|
712
|
-
id, operation, status, requested_at, started_at, metadata_json
|
|
713
|
-
)
|
|
714
|
-
VALUES (?, ?, 'running', ?, ?, ?)
|
|
715
|
-
""",
|
|
716
|
-
(
|
|
717
|
-
op_id,
|
|
718
|
-
"rebuild_full" if full else "rebuild_incremental",
|
|
719
|
-
requested_at,
|
|
720
|
-
requested_at,
|
|
721
|
-
_json(
|
|
722
|
-
{
|
|
723
|
-
"include_nodes": include_nodes,
|
|
724
|
-
"include_chunks": include_chunks,
|
|
725
|
-
}
|
|
726
|
-
),
|
|
727
|
-
),
|
|
728
|
-
)
|
|
729
|
-
if full:
|
|
730
|
-
filters = []
|
|
731
|
-
if include_nodes:
|
|
732
|
-
filters.append("'node'")
|
|
733
|
-
if include_chunks:
|
|
734
|
-
filters.append("'chunk'")
|
|
735
|
-
if filters:
|
|
736
|
-
conn.execute(
|
|
737
|
-
f"DELETE FROM vector_embeddings WHERE item_type IN ({','.join(filters)})"
|
|
738
|
-
)
|
|
739
|
-
items = self._iter_vector_source_items(
|
|
740
|
-
conn,
|
|
741
|
-
include_nodes=include_nodes,
|
|
742
|
-
include_chunks=include_chunks,
|
|
743
|
-
)
|
|
744
|
-
indexed = skipped = 0
|
|
745
|
-
for item in items:
|
|
746
|
-
changed = self._upsert_vector_item(conn, **item)
|
|
747
|
-
if changed:
|
|
748
|
-
indexed += 1
|
|
749
|
-
else:
|
|
750
|
-
skipped += 1
|
|
751
|
-
duration_ms = round((time.perf_counter() - started) * 1000, 2)
|
|
752
|
-
conn.execute(
|
|
753
|
-
"""
|
|
754
|
-
UPDATE vector_index_operations
|
|
755
|
-
SET status='completed', completed_at=?, items_total=?,
|
|
756
|
-
items_indexed=?, items_skipped=?, metadata_json=?
|
|
757
|
-
WHERE id=?
|
|
758
|
-
""",
|
|
759
|
-
(
|
|
760
|
-
_now(),
|
|
761
|
-
len(items),
|
|
762
|
-
indexed,
|
|
763
|
-
skipped,
|
|
764
|
-
_json(
|
|
765
|
-
{
|
|
766
|
-
"include_nodes": include_nodes,
|
|
767
|
-
"include_chunks": include_chunks,
|
|
768
|
-
"duration_ms": duration_ms,
|
|
769
|
-
"embedding_model": self._embedding_model.model_id,
|
|
770
|
-
"embedding_dim": self._embedding_model.dim,
|
|
771
|
-
}
|
|
772
|
-
),
|
|
773
|
-
op_id,
|
|
774
|
-
),
|
|
775
|
-
)
|
|
776
|
-
return {
|
|
777
|
-
"status": "completed",
|
|
778
|
-
"operation_id": op_id,
|
|
779
|
-
"full": bool(full),
|
|
780
|
-
"items_total": len(items),
|
|
781
|
-
"items_indexed": indexed,
|
|
782
|
-
"items_skipped": skipped,
|
|
783
|
-
"duration_ms": duration_ms,
|
|
784
|
-
"embedding_model": self._embedding_model.model_id,
|
|
785
|
-
"embedding_dim": self._embedding_model.dim,
|
|
786
|
-
}
|
|
787
|
-
except Exception as exc:
|
|
788
|
-
duration_ms = round((time.perf_counter() - started) * 1000, 2)
|
|
789
|
-
with self._connect() as conn:
|
|
790
|
-
conn.execute(
|
|
791
|
-
"""
|
|
792
|
-
INSERT INTO vector_index_operations(
|
|
793
|
-
id, operation, status, requested_at, started_at, completed_at,
|
|
794
|
-
error_message, metadata_json
|
|
795
|
-
)
|
|
796
|
-
VALUES (?, ?, 'failed', ?, ?, ?, ?, ?)
|
|
797
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
798
|
-
status='failed',
|
|
799
|
-
completed_at=excluded.completed_at,
|
|
800
|
-
error_message=excluded.error_message,
|
|
801
|
-
metadata_json=excluded.metadata_json
|
|
802
|
-
""",
|
|
803
|
-
(
|
|
804
|
-
op_id,
|
|
805
|
-
"rebuild_full" if full else "rebuild_incremental",
|
|
806
|
-
requested_at,
|
|
807
|
-
requested_at,
|
|
808
|
-
_now(),
|
|
809
|
-
str(exc),
|
|
810
|
-
_json({"duration_ms": duration_ms}),
|
|
811
|
-
),
|
|
812
|
-
)
|
|
813
|
-
raise
|
|
814
|
-
|
|
815
|
-
def index_status(self) -> Dict[str, Any]:
|
|
816
|
-
storage_capabilities = None
|
|
817
|
-
try:
|
|
818
|
-
storage_capabilities = self.storage_engine.capabilities().as_dict()
|
|
819
|
-
except Exception as exc:
|
|
820
|
-
storage_capabilities = {
|
|
821
|
-
"engine": "sqlite",
|
|
822
|
-
"available": False,
|
|
823
|
-
"reason": str(exc),
|
|
824
|
-
}
|
|
825
|
-
with self._connect() as conn:
|
|
826
|
-
vector_counts = {
|
|
827
|
-
row["item_type"]: row["count"]
|
|
828
|
-
for row in conn.execute(
|
|
829
|
-
"SELECT item_type, COUNT(*) AS count FROM vector_embeddings GROUP BY item_type"
|
|
830
|
-
)
|
|
831
|
-
}
|
|
832
|
-
source_items = self._iter_vector_source_items(conn)
|
|
833
|
-
vector_rows = {
|
|
834
|
-
row["item_id"]: row
|
|
835
|
-
for row in conn.execute(
|
|
836
|
-
"""
|
|
837
|
-
SELECT item_id, text_hash, embedding_dim, embedding_model, indexed_at
|
|
838
|
-
FROM vector_embeddings
|
|
839
|
-
"""
|
|
840
|
-
).fetchall()
|
|
841
|
-
}
|
|
842
|
-
latest_rows = conn.execute(
|
|
843
|
-
"""
|
|
844
|
-
SELECT id, operation, status, requested_at, started_at, completed_at,
|
|
845
|
-
items_total, items_indexed, items_skipped, error_message, metadata_json
|
|
846
|
-
FROM vector_index_operations
|
|
847
|
-
ORDER BY requested_at DESC, id DESC
|
|
848
|
-
LIMIT 5
|
|
849
|
-
"""
|
|
850
|
-
).fetchall()
|
|
851
|
-
missing = stale = ready = 0
|
|
852
|
-
for item in source_items:
|
|
853
|
-
vector_row = vector_rows.get(item["item_id"])
|
|
854
|
-
expected_hash = _sha256_text(_clean_text(item["text"]))
|
|
855
|
-
if not vector_row:
|
|
856
|
-
missing += 1
|
|
857
|
-
elif (
|
|
858
|
-
vector_row["text_hash"] != expected_hash
|
|
859
|
-
or vector_row["embedding_dim"] != self._embedding_model.dim
|
|
860
|
-
or vector_row["embedding_model"] != self._embedding_model.model_id
|
|
861
|
-
):
|
|
862
|
-
stale += 1
|
|
863
|
-
else:
|
|
864
|
-
ready += 1
|
|
865
|
-
pending = missing + stale
|
|
866
|
-
return {
|
|
867
|
-
"status": "ready" if pending == 0 else "needs_reindex",
|
|
868
|
-
"storage": {
|
|
869
|
-
"db_path": str(self.db_path),
|
|
870
|
-
"backend": "sqlite",
|
|
871
|
-
"embedding_model": self._embedding_model.model_id,
|
|
872
|
-
"embedding_dim": self._embedding_model.dim,
|
|
873
|
-
# Honest capability report: trigram FTS5 keyword index, or
|
|
874
|
-
# LIKE-scan fallback when this SQLite build lacks it.
|
|
875
|
-
"fts_enabled": bool(getattr(self, "_fts_enabled", False)),
|
|
876
|
-
"engine": storage_capabilities,
|
|
877
|
-
"vector_search_backend": (
|
|
878
|
-
storage_capabilities.get("vector_backend")
|
|
879
|
-
if isinstance(storage_capabilities, dict)
|
|
880
|
-
else "bruteforce-cosine"
|
|
881
|
-
),
|
|
882
|
-
"vector_search_mode": (
|
|
883
|
-
(storage_capabilities.get("metadata") or {}).get("vector_mode")
|
|
884
|
-
if isinstance(storage_capabilities, dict)
|
|
885
|
-
else "fallback"
|
|
886
|
-
),
|
|
887
|
-
"sqlite_vec_ann_available": (
|
|
888
|
-
bool((storage_capabilities.get("metadata") or {}).get("sqlite_vec_ann_available"))
|
|
889
|
-
if isinstance(storage_capabilities, dict)
|
|
890
|
-
else False
|
|
891
|
-
),
|
|
892
|
-
},
|
|
893
|
-
"source_items": len(source_items),
|
|
894
|
-
"indexed_items": sum(vector_counts.values()),
|
|
895
|
-
"ready_items": ready,
|
|
896
|
-
"missing_items": missing,
|
|
897
|
-
"stale_items": stale,
|
|
898
|
-
"pending_items": pending,
|
|
899
|
-
"by_item_type": vector_counts,
|
|
900
|
-
"operations": [
|
|
901
|
-
{
|
|
902
|
-
"id": row["id"],
|
|
903
|
-
"operation": row["operation"],
|
|
904
|
-
"status": row["status"],
|
|
905
|
-
"requested_at": row["requested_at"],
|
|
906
|
-
"started_at": row["started_at"],
|
|
907
|
-
"completed_at": row["completed_at"],
|
|
908
|
-
"items_total": row["items_total"],
|
|
909
|
-
"items_indexed": row["items_indexed"],
|
|
910
|
-
"items_skipped": row["items_skipped"],
|
|
911
|
-
"error_message": row["error_message"],
|
|
912
|
-
"metadata": _safe_loads(row["metadata_json"]),
|
|
913
|
-
}
|
|
914
|
-
for row in latest_rows
|
|
915
|
-
],
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
def vector_search(
|
|
919
|
-
self,
|
|
920
|
-
query: str,
|
|
921
|
-
*,
|
|
922
|
-
limit: int = 30,
|
|
923
|
-
min_score: float = 0.0,
|
|
924
|
-
max_candidates: int = 10_000,
|
|
925
|
-
) -> Dict[str, Any]:
|
|
926
|
-
query = str(query or "").strip()
|
|
927
|
-
limit = max(1, min(int(limit or 30), 100))
|
|
928
|
-
min_score = float(min_score or 0.0)
|
|
929
|
-
if not query:
|
|
930
|
-
return {"query": query, "matches": []}
|
|
931
|
-
query_vector = self._embedding_model.embed(query)
|
|
932
|
-
max_candidates = max(limit, min(int(max_candidates or 10_000), 50_000))
|
|
933
|
-
with self._connect() as conn:
|
|
934
|
-
rows = conn.execute(
|
|
935
|
-
"""
|
|
936
|
-
SELECT
|
|
937
|
-
ve.item_id, ve.item_type, ve.source_node, ve.embedding,
|
|
938
|
-
ve.embedding_dim, ve.embedding_model, ve.metadata_json AS vector_metadata,
|
|
939
|
-
n.type AS node_type, n.title AS node_title, n.summary AS node_summary,
|
|
940
|
-
n.metadata_json AS node_metadata, n.updated_at AS node_updated_at,
|
|
941
|
-
c.text AS chunk_text, c.source_node AS parent_node_id,
|
|
942
|
-
pn.type AS parent_type, pn.title AS parent_title,
|
|
943
|
-
pn.summary AS parent_summary, pn.metadata_json AS parent_metadata,
|
|
944
|
-
pn.updated_at AS parent_updated_at
|
|
945
|
-
FROM vector_embeddings ve
|
|
946
|
-
LEFT JOIN nodes n ON n.id=ve.source_node
|
|
947
|
-
LEFT JOIN chunks c ON c.id=ve.item_id
|
|
948
|
-
LEFT JOIN nodes pn ON pn.id=c.source_node
|
|
949
|
-
WHERE ve.embedding_model=? AND ve.embedding_dim=?
|
|
950
|
-
ORDER BY ve.indexed_at DESC
|
|
951
|
-
LIMIT ?
|
|
952
|
-
""",
|
|
953
|
-
(
|
|
954
|
-
self._embedding_model.model_id,
|
|
955
|
-
self._embedding_model.dim,
|
|
956
|
-
max_candidates,
|
|
957
|
-
),
|
|
958
|
-
).fetchall()
|
|
959
|
-
scored = []
|
|
960
|
-
for row in rows:
|
|
961
|
-
vector = self._embedding_model.decode(
|
|
962
|
-
row["embedding"], row["embedding_dim"]
|
|
963
|
-
)
|
|
964
|
-
score = self._embedding_model.similarity(query_vector, vector)
|
|
965
|
-
if score < min_score:
|
|
966
|
-
continue
|
|
967
|
-
is_chunk = row["item_type"] == "chunk"
|
|
968
|
-
summary = (
|
|
969
|
-
row["chunk_text"]
|
|
970
|
-
if is_chunk and row["chunk_text"]
|
|
971
|
-
else row["node_summary"]
|
|
972
|
-
)
|
|
973
|
-
parent_metadata = _safe_loads(row["parent_metadata"])
|
|
974
|
-
node_metadata = _safe_loads(row["node_metadata"])
|
|
975
|
-
scored.append(
|
|
976
|
-
{
|
|
977
|
-
"id": row["item_id"],
|
|
978
|
-
"node_id": row["parent_node_id"]
|
|
979
|
-
if is_chunk and row["parent_node_id"]
|
|
980
|
-
else row["source_node"],
|
|
981
|
-
"item_type": row["item_type"],
|
|
982
|
-
"type": "Chunk" if is_chunk else row["node_type"],
|
|
983
|
-
"title": row["parent_title"]
|
|
984
|
-
if is_chunk and row["parent_title"]
|
|
985
|
-
else row["node_title"],
|
|
986
|
-
"summary": _clean_text(summary or "")[:1000],
|
|
987
|
-
"score": round(float(score), 6),
|
|
988
|
-
"metadata": {
|
|
989
|
-
**(parent_metadata if is_chunk else node_metadata),
|
|
990
|
-
"vector": _safe_loads(row["vector_metadata"]),
|
|
991
|
-
"parent_node_id": row["parent_node_id"],
|
|
992
|
-
"parent_type": row["parent_type"],
|
|
993
|
-
},
|
|
994
|
-
"updated_at": row["parent_updated_at"]
|
|
995
|
-
if is_chunk and row["parent_updated_at"]
|
|
996
|
-
else row["node_updated_at"],
|
|
997
|
-
}
|
|
998
|
-
)
|
|
999
|
-
scored.sort(
|
|
1000
|
-
key=lambda item: (item["score"], item.get("updated_at") or ""), reverse=True
|
|
1001
|
-
)
|
|
1002
|
-
return {
|
|
1003
|
-
"query": query,
|
|
1004
|
-
"embedding_model": self._embedding_model.model_id,
|
|
1005
|
-
"embedding_dim": self._embedding_model.dim,
|
|
1006
|
-
"matches": scored[:limit],
|
|
1007
|
-
}
|
|
668
|
+
]
|
|
669
|
+
edges = list(edges_by_id.values())
|
|
670
|
+
if allowed_workspaces is not None:
|
|
671
|
+
nodes = self.filter_scoped_nodes(nodes, allowed_workspaces)
|
|
672
|
+
kept = {node.get("id") for node in nodes}
|
|
673
|
+
edges = [edge for edge in edges if edge.get("from") in kept and edge.get("to") in kept]
|
|
674
|
+
return {"root": node_id, "depth": depth, "nodes": nodes, "edges": edges}
|
|
1008
675
|
|
|
1009
676
|
def delete_conversation(self, conversation_id: str) -> Dict[str, Any]:
|
|
1010
677
|
conversation_id = str(conversation_id or "").strip()
|
|
@@ -1141,201 +808,3 @@ class KnowledgeGraphRetrievalMixin:
|
|
|
1141
808
|
"local_file_status": local_file_status,
|
|
1142
809
|
"v2": v2,
|
|
1143
810
|
}
|
|
1144
|
-
|
|
1145
|
-
def search_for_document_generation(
|
|
1146
|
-
self, query: str, limit: int = 10
|
|
1147
|
-
) -> List[Dict[str, Any]]:
|
|
1148
|
-
"""Hybrid retrieval optimized for document generation.
|
|
1149
|
-
|
|
1150
|
-
Scoring: 0.5*text_relevance + 0.3*graph_relationship + 0.2*recency
|
|
1151
|
-
Returns nodes with rich context for document generation prompts.
|
|
1152
|
-
"""
|
|
1153
|
-
query = str(query or "").strip()
|
|
1154
|
-
if not query:
|
|
1155
|
-
return []
|
|
1156
|
-
limit = max(1, min(int(limit or 10), 50))
|
|
1157
|
-
terms = _topic_candidates(query, limit=12)
|
|
1158
|
-
now = datetime.now()
|
|
1159
|
-
nt, et = self._read_tables()
|
|
1160
|
-
|
|
1161
|
-
with self._connect() as conn:
|
|
1162
|
-
candidate_rows = []
|
|
1163
|
-
seen_ids = set()
|
|
1164
|
-
|
|
1165
|
-
if query:
|
|
1166
|
-
q = f"%{query}%"
|
|
1167
|
-
rows = conn.execute(
|
|
1168
|
-
f"""
|
|
1169
|
-
SELECT id, type, title, summary, metadata_json, updated_at
|
|
1170
|
-
FROM {nt}
|
|
1171
|
-
WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
|
|
1172
|
-
AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
|
|
1173
|
-
'Spreadsheet', 'Image', 'ImageText', 'Chat',
|
|
1174
|
-
'Decision', 'Task', 'Concept', 'Feature',
|
|
1175
|
-
'Page', 'Slide')
|
|
1176
|
-
ORDER BY updated_at DESC, id ASC
|
|
1177
|
-
LIMIT ?
|
|
1178
|
-
""",
|
|
1179
|
-
(q, q, q, limit * 5),
|
|
1180
|
-
).fetchall()
|
|
1181
|
-
for row in rows:
|
|
1182
|
-
if row["id"] not in seen_ids:
|
|
1183
|
-
seen_ids.add(row["id"])
|
|
1184
|
-
candidate_rows.append(row)
|
|
1185
|
-
|
|
1186
|
-
for term in terms:
|
|
1187
|
-
t = f"%{term}%"
|
|
1188
|
-
rows = conn.execute(
|
|
1189
|
-
f"""
|
|
1190
|
-
SELECT id, type, title, summary, metadata_json, updated_at
|
|
1191
|
-
FROM {nt}
|
|
1192
|
-
WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
|
|
1193
|
-
AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
|
|
1194
|
-
'Spreadsheet', 'Image', 'ImageText', 'Chat',
|
|
1195
|
-
'Decision', 'Task', 'Concept', 'Feature',
|
|
1196
|
-
'Page', 'Slide')
|
|
1197
|
-
ORDER BY updated_at DESC, id ASC
|
|
1198
|
-
LIMIT ?
|
|
1199
|
-
""",
|
|
1200
|
-
(t, t, t, limit * 3),
|
|
1201
|
-
).fetchall()
|
|
1202
|
-
for row in rows:
|
|
1203
|
-
if row["id"] not in seen_ids:
|
|
1204
|
-
seen_ids.add(row["id"])
|
|
1205
|
-
candidate_rows.append(row)
|
|
1206
|
-
|
|
1207
|
-
scored_results = []
|
|
1208
|
-
for row in candidate_rows:
|
|
1209
|
-
haystack = (
|
|
1210
|
-
f"{row['title']} {row['summary']} {row['metadata_json']}".lower()
|
|
1211
|
-
)
|
|
1212
|
-
|
|
1213
|
-
text_hits = sum(1 for term in terms if term.lower() in haystack)
|
|
1214
|
-
text_score = min(1.0, text_hits / max(len(terms), 1))
|
|
1215
|
-
|
|
1216
|
-
edge_count = conn.execute(
|
|
1217
|
-
f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
|
|
1218
|
-
(row["id"], row["id"]),
|
|
1219
|
-
).fetchone()["c"]
|
|
1220
|
-
graph_score = min(1.0, math.log1p(edge_count) / 4.0)
|
|
1221
|
-
|
|
1222
|
-
recency = _recency_score(
|
|
1223
|
-
row["updated_at"], now=now, half_life_days=14.0
|
|
1224
|
-
)
|
|
1225
|
-
|
|
1226
|
-
doc_type_boost = (
|
|
1227
|
-
1.2
|
|
1228
|
-
if row["type"]
|
|
1229
|
-
in (
|
|
1230
|
-
"Document",
|
|
1231
|
-
"File",
|
|
1232
|
-
"SlideDeck",
|
|
1233
|
-
"Decision",
|
|
1234
|
-
)
|
|
1235
|
-
else 1.0
|
|
1236
|
-
)
|
|
1237
|
-
|
|
1238
|
-
hybrid_score = (
|
|
1239
|
-
0.5 * text_score + 0.3 * graph_score + 0.2 * recency
|
|
1240
|
-
) * doc_type_boost
|
|
1241
|
-
|
|
1242
|
-
meta = _safe_loads(row["metadata_json"])
|
|
1243
|
-
neighbor_concepts = []
|
|
1244
|
-
neighbor_rows = conn.execute(
|
|
1245
|
-
f"""
|
|
1246
|
-
SELECT n.title, n.type FROM {et} e
|
|
1247
|
-
JOIN {nt} n ON n.id = CASE WHEN e.from_node = ? THEN e.to_node ELSE e.from_node END
|
|
1248
|
-
WHERE (e.from_node = ? OR e.to_node = ?)
|
|
1249
|
-
AND n.type IN ('Concept', 'Feature', 'Decision', 'Task')
|
|
1250
|
-
LIMIT 8
|
|
1251
|
-
""",
|
|
1252
|
-
(row["id"], row["id"], row["id"]),
|
|
1253
|
-
).fetchall()
|
|
1254
|
-
for nr in neighbor_rows:
|
|
1255
|
-
neighbor_concepts.append({"title": nr["title"], "type": nr["type"]})
|
|
1256
|
-
|
|
1257
|
-
scored_results.append(
|
|
1258
|
-
{
|
|
1259
|
-
"id": row["id"],
|
|
1260
|
-
"type": row["type"],
|
|
1261
|
-
"title": row["title"],
|
|
1262
|
-
"summary": row["summary"],
|
|
1263
|
-
"metadata": meta,
|
|
1264
|
-
"updated_at": row["updated_at"],
|
|
1265
|
-
"hybrid_score": round(hybrid_score, 4),
|
|
1266
|
-
"scores": {
|
|
1267
|
-
"text": round(text_score, 4),
|
|
1268
|
-
"graph": round(graph_score, 4),
|
|
1269
|
-
"recency": round(recency, 4),
|
|
1270
|
-
},
|
|
1271
|
-
"related_concepts": neighbor_concepts,
|
|
1272
|
-
}
|
|
1273
|
-
)
|
|
1274
|
-
|
|
1275
|
-
scored_results.sort(key=lambda x: x["hybrid_score"], reverse=True)
|
|
1276
|
-
return scored_results[:limit]
|
|
1277
|
-
|
|
1278
|
-
def multi_hop_context(
|
|
1279
|
-
self, node_ids: List[str], max_hops: int = 2
|
|
1280
|
-
) -> Dict[str, Any]:
|
|
1281
|
-
"""Multi-hop graph traversal from seed nodes for richer context."""
|
|
1282
|
-
visited_nodes = set()
|
|
1283
|
-
visited_edges = set()
|
|
1284
|
-
all_nodes = []
|
|
1285
|
-
all_edges = []
|
|
1286
|
-
frontier = set(node_ids)
|
|
1287
|
-
nt, et = self._read_tables()
|
|
1288
|
-
|
|
1289
|
-
with self._connect() as conn:
|
|
1290
|
-
for hop in range(max_hops):
|
|
1291
|
-
if not frontier:
|
|
1292
|
-
break
|
|
1293
|
-
next_frontier = set()
|
|
1294
|
-
for nid in frontier:
|
|
1295
|
-
if nid in visited_nodes:
|
|
1296
|
-
continue
|
|
1297
|
-
visited_nodes.add(nid)
|
|
1298
|
-
row = conn.execute(
|
|
1299
|
-
f"SELECT id, type, title, summary, metadata_json, updated_at FROM {nt} WHERE id=?",
|
|
1300
|
-
(nid,),
|
|
1301
|
-
).fetchone()
|
|
1302
|
-
if row:
|
|
1303
|
-
all_nodes.append(
|
|
1304
|
-
{
|
|
1305
|
-
"id": row["id"],
|
|
1306
|
-
"type": row["type"],
|
|
1307
|
-
"title": row["title"],
|
|
1308
|
-
"summary": row["summary"],
|
|
1309
|
-
"metadata": _safe_loads(row["metadata_json"]),
|
|
1310
|
-
"hop": hop,
|
|
1311
|
-
}
|
|
1312
|
-
)
|
|
1313
|
-
edge_rows = conn.execute(
|
|
1314
|
-
f"""
|
|
1315
|
-
SELECT id, from_node, to_node, type, weight
|
|
1316
|
-
FROM {et} WHERE from_node=? OR to_node=?
|
|
1317
|
-
ORDER BY id ASC
|
|
1318
|
-
""",
|
|
1319
|
-
(nid, nid),
|
|
1320
|
-
).fetchall()
|
|
1321
|
-
for er in edge_rows:
|
|
1322
|
-
if er["id"] not in visited_edges:
|
|
1323
|
-
visited_edges.add(er["id"])
|
|
1324
|
-
all_edges.append(
|
|
1325
|
-
{
|
|
1326
|
-
"from": er["from_node"],
|
|
1327
|
-
"to": er["to_node"],
|
|
1328
|
-
"type": er["type"],
|
|
1329
|
-
"weight": er["weight"],
|
|
1330
|
-
}
|
|
1331
|
-
)
|
|
1332
|
-
other = (
|
|
1333
|
-
er["to_node"]
|
|
1334
|
-
if er["from_node"] == nid
|
|
1335
|
-
else er["from_node"]
|
|
1336
|
-
)
|
|
1337
|
-
if other not in visited_nodes:
|
|
1338
|
-
next_frontier.add(other)
|
|
1339
|
-
frontier = next_frontier
|
|
1340
|
-
|
|
1341
|
-
return {"nodes": all_nodes, "edges": all_edges}
|