ltcai 11.1.0 → 11.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -54
- package/docs/CHANGELOG.md +94 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/FEATURE_AUDIT_v11.2.0.md +393 -0
- package/docs/LAYOUT_REBUILD_SPEC.md +9 -1
- package/docs/MULTI_AGENT_RUNTIME.md +1 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +6 -2
- package/docs/PERMISSION_MODE.md +1 -1
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/architecture.md +6 -2
- package/docs/kg-schema.md +2 -2
- package/docs/v11.3.0_PLAN.md +202 -0
- package/docs/v11.4.0_RUST_FOUNDATION_PLAN.md +176 -0
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/gates.py +125 -0
- package/lattice_brain/graph/_kg_common/__init__.py +287 -0
- package/lattice_brain/graph/_kg_common/extraction.py +516 -0
- package/lattice_brain/graph/_kg_common/relations.py +161 -0
- package/lattice_brain/graph/_kg_common/text.py +479 -0
- package/lattice_brain/graph/discovery_index/__init__.py +35 -0
- package/lattice_brain/graph/discovery_index/cleanup.py +182 -0
- package/lattice_brain/graph/discovery_index/extract.py +137 -0
- package/lattice_brain/graph/discovery_index/scan.py +411 -0
- package/lattice_brain/graph/discovery_index/upsert.py +495 -0
- package/lattice_brain/graph/fusion.py +35 -4
- package/lattice_brain/graph/projection/__init__.py +42 -0
- package/lattice_brain/graph/projection/curation.py +500 -0
- package/lattice_brain/graph/{projection.py → projection/v2_schema.py} +81 -485
- package/lattice_brain/graph/retrieval/__init__.py +54 -0
- package/lattice_brain/graph/retrieval/context.py +197 -0
- package/lattice_brain/graph/retrieval/graph_view.py +319 -0
- package/lattice_brain/graph/retrieval/hybrid.py +488 -0
- package/lattice_brain/graph/retrieval/maintenance.py +121 -0
- package/lattice_brain/graph/retrieval/signals.py +95 -0
- package/lattice_brain/graph/retrieval_vector/__init__.py +42 -0
- package/lattice_brain/graph/retrieval_vector/fingerprint.py +97 -0
- package/lattice_brain/graph/retrieval_vector/indexing.py +347 -0
- package/lattice_brain/graph/retrieval_vector/search.py +560 -0
- package/lattice_brain/graph/retrieval_vector/status.py +374 -0
- package/lattice_brain/graph/schema.py +9 -0
- package/lattice_brain/graph/store.py +9 -0
- package/lattice_brain/graph/vector_index/selector.py +32 -2
- package/lattice_brain/ingestion/__init__.py +130 -0
- package/lattice_brain/ingestion/_contract.py +90 -0
- package/lattice_brain/ingestion/constants.py +127 -0
- package/lattice_brain/ingestion/folder_scan.py +57 -0
- package/lattice_brain/ingestion/folders.py +258 -0
- package/lattice_brain/ingestion/hashing.py +26 -0
- package/lattice_brain/ingestion/jobs_api.py +107 -0
- package/lattice_brain/ingestion/models.py +80 -0
- package/lattice_brain/ingestion/pipeline.py +486 -0
- package/lattice_brain/ingestion/quality.py +209 -0
- package/lattice_brain/ingestion/routing.py +295 -0
- package/lattice_brain/multimodal/__init__.py +164 -0
- package/lattice_brain/multimodal/audio.py +77 -0
- package/lattice_brain/multimodal/common.py +118 -0
- package/lattice_brain/{multimodal.py → multimodal/images.py} +22 -262
- package/lattice_brain/multimodal/ports.py +169 -0
- package/lattice_brain/multimodal/video.py +410 -0
- package/lattice_brain/portability/__init__.py +90 -0
- package/lattice_brain/portability/_contract.py +42 -0
- package/lattice_brain/portability/backups.py +338 -0
- package/lattice_brain/portability/bundles.py +136 -0
- package/lattice_brain/portability/constants.py +93 -0
- package/lattice_brain/portability/fsops.py +138 -0
- package/lattice_brain/portability/service.py +41 -0
- package/lattice_brain/portability/sharing.py +714 -0
- package/lattice_brain/runtime/__init__.py +1 -1
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/lattice_brain/sealed_box.py +244 -0
- package/lattice_brain/synthesis.py +24 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/brain_intelligence.py +4 -0
- package/latticeai/api/chat.py +11 -0
- package/latticeai/api/chat_helpers.py +16 -3
- package/latticeai/api/chat_hybrid.py +32 -1
- package/latticeai/api/chronicle.py +63 -0
- package/latticeai/api/features.py +70 -0
- package/latticeai/api/local_files.py +102 -0
- package/latticeai/api/portability.py +39 -4
- package/latticeai/api/review_queue.py +126 -0
- package/latticeai/api/search.py +16 -2
- package/latticeai/core/agent/__init__.py +93 -0
- package/latticeai/core/agent/_contract.py +79 -0
- package/latticeai/core/agent/context.py +57 -0
- package/latticeai/core/agent/deps.py +125 -0
- package/latticeai/core/agent/execution.py +622 -0
- package/latticeai/core/agent/planning.py +145 -0
- package/latticeai/core/agent/recovery.py +157 -0
- package/latticeai/core/agent/runtime.py +210 -0
- package/latticeai/core/agent/verification.py +231 -0
- package/latticeai/core/config.py +4 -1
- package/latticeai/core/context_builder.py +6 -3
- package/latticeai/core/embedding_providers/__init__.py +151 -0
- package/latticeai/core/embedding_providers/base.py +199 -0
- package/latticeai/core/embedding_providers/captions.py +162 -0
- package/latticeai/core/embedding_providers/profiles.py +126 -0
- package/latticeai/core/embedding_providers/text.py +350 -0
- package/latticeai/core/embedding_providers/vision.py +352 -0
- package/latticeai/core/file_generation/__init__.py +115 -0
- package/latticeai/core/file_generation/bundles.py +76 -0
- package/latticeai/core/file_generation/extraction.py +154 -0
- package/latticeai/core/file_generation/inference.py +235 -0
- package/latticeai/core/file_generation/orchestration.py +152 -0
- package/latticeai/core/file_generation/prompting.py +117 -0
- package/latticeai/core/file_generation/repair.py +114 -0
- package/latticeai/core/file_generation/sanitize.py +61 -0
- package/latticeai/core/file_generation/validation.py +201 -0
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/messages.py +152 -0
- package/latticeai/core/model_compat.py +73 -2
- package/latticeai/core/workspace_os_constants.py +1 -1
- package/latticeai/integrations/telegram_bot/__init__.py +123 -0
- package/latticeai/integrations/telegram_bot/__main__.py +17 -0
- package/latticeai/integrations/telegram_bot/config.py +86 -0
- package/latticeai/integrations/telegram_bot/dispatch.py +311 -0
- package/latticeai/integrations/telegram_bot/flows.py +478 -0
- package/latticeai/integrations/telegram_bot/helpers.py +322 -0
- package/latticeai/integrations/telegram_bot/screens.py +394 -0
- package/latticeai/models/model_providers.py +12 -4
- package/latticeai/models/router/__init__.py +88 -0
- package/latticeai/models/router/_contract.py +66 -0
- package/latticeai/models/router/branding.py +56 -0
- package/latticeai/models/router/catalog.py +69 -0
- package/latticeai/models/router/documents.py +199 -0
- package/latticeai/models/router/errors.py +37 -0
- package/latticeai/models/router/generation.py +258 -0
- package/latticeai/models/router/loading.py +291 -0
- package/latticeai/models/router/local_models.py +85 -0
- package/latticeai/models/router/registry.py +147 -0
- package/latticeai/runtime/build_phases/__init__.py +82 -0
- package/latticeai/runtime/build_phases/features.py +407 -0
- package/latticeai/runtime/build_phases/foundation.py +555 -0
- package/latticeai/runtime/build_phases/web.py +492 -0
- package/latticeai/runtime/chat_wiring.py +4 -0
- package/latticeai/runtime/feature_toggle_wiring.py +163 -0
- package/latticeai/runtime/router_registration.py +11 -0
- package/latticeai/runtime/runtime_context.py +1 -0
- package/latticeai/services/app_context.py +8 -0
- package/latticeai/services/architecture_readiness.py +48 -19
- package/latticeai/services/automation_intelligence.py +22 -2
- package/latticeai/services/brain_intelligence/__init__.py +58 -0
- package/latticeai/services/brain_intelligence/_contract.py +71 -0
- package/latticeai/services/brain_intelligence/consistency.py +193 -0
- package/latticeai/services/brain_intelligence/constants.py +47 -0
- package/latticeai/services/brain_intelligence/digest.py +258 -0
- package/latticeai/services/brain_intelligence/health.py +331 -0
- package/latticeai/services/brain_intelligence/proposals.py +264 -0
- package/latticeai/services/brain_intelligence/sampling.py +84 -0
- package/latticeai/services/brain_intelligence/service.py +48 -0
- package/latticeai/services/chronicle.py +557 -0
- package/latticeai/services/command_center.py +10 -4
- package/latticeai/services/feature_toggles.py +502 -0
- package/latticeai/services/folder_watch.py +122 -1
- package/latticeai/services/hybrid_chat.py +56 -5
- package/latticeai/services/interop_bridges.py +978 -0
- package/latticeai/services/memory_service/__init__.py +52 -0
- package/latticeai/services/memory_service/_contract.py +100 -0
- package/latticeai/services/memory_service/brief.py +431 -0
- package/latticeai/services/memory_service/constants.py +57 -0
- package/latticeai/services/memory_service/maintenance.py +138 -0
- package/latticeai/services/memory_service/manager.py +186 -0
- package/latticeai/services/memory_service/proof.py +136 -0
- package/latticeai/services/memory_service/recall.py +225 -0
- package/latticeai/services/memory_service/service.py +48 -0
- package/latticeai/services/memory_service/stores.py +110 -0
- package/latticeai/services/model_capability_registry.py +434 -261
- package/latticeai/services/model_catalog.py +95 -61
- package/latticeai/services/model_recommendation.py +18 -11
- package/latticeai/services/model_runtime/__init__.py +322 -0
- package/latticeai/services/model_runtime/cloud.py +87 -0
- package/latticeai/services/model_runtime/download.py +282 -0
- package/latticeai/services/model_runtime/engines.py +341 -0
- package/latticeai/services/model_runtime/loading.py +178 -0
- package/latticeai/services/model_runtime/service.py +129 -0
- package/latticeai/services/model_runtime/state.py +131 -0
- package/latticeai/services/model_runtime/status.py +255 -0
- package/latticeai/services/multimodal_ports.py +26 -1
- package/latticeai/services/obsidian_bridge.py +16 -25
- package/latticeai/services/product_readiness.py +15 -7
- package/latticeai/services/search_service.py +149 -2
- package/latticeai/services/tool_dispatch.py +4 -0
- package/latticeai/setup/auto_setup.py +27 -30
- package/latticeai/setup/wizard/__init__.py +126 -0
- package/latticeai/setup/wizard/catalog.py +172 -0
- package/latticeai/setup/wizard/detect.py +323 -0
- package/latticeai/setup/wizard/install.py +348 -0
- package/latticeai/setup/wizard/paths.py +168 -0
- package/latticeai/setup/wizard/plans.py +74 -0
- package/latticeai/setup/wizard/recommend.py +320 -0
- package/package.json +6 -2
- package/scripts/bump_version.py +14 -0
- package/scripts/capture_release_evidence.mjs +33 -21
- package/scripts/check_current_release_docs.mjs +1 -1
- package/scripts/check_i18n_namespace_coverage.mjs +41 -4
- package/scripts/check_max_file_lines.mjs +102 -0
- package/scripts/check_release_evidence_bound.mjs +30 -15
- package/scripts/check_screenshot_pixel_delta.py +34 -4
- package/scripts/check_server_i18n.mjs +2 -0
- package/scripts/generate_rust_parity_fixtures.py +562 -0
- package/scripts/lib/mock_server_fingerprint.mjs +94 -0
- package/scripts/release_screen_claims.json +44 -2
- package/scripts/verify_hf_model_registry.py +253 -218
- package/src-tauri/Cargo.lock +361 -3
- package/src-tauri/Cargo.toml +6 -1
- package/src-tauri/src/backend.rs +349 -0
- package/src-tauri/src/folder.rs +33 -0
- package/src-tauri/src/main.rs +97 -399
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +41 -37
- package/static/app/assets/Act-yYpYnn0v.js +1 -0
- package/static/app/assets/AdminConsole-DL3Cr5pL.js +1 -0
- package/static/app/assets/{Brain-CzCsI1mi.js → Brain-C1HBN0Wf.js} +2 -2
- package/static/app/assets/BrainHome-DoXRhUUC.js +2 -0
- package/static/app/assets/BrainSignals-6yR6ir5t.js +1 -0
- package/static/app/assets/Capture-CFIRsFNE.js +1 -0
- package/static/app/assets/Chronicle-BZbEgiwN.js +1 -0
- package/static/app/assets/CommandPalette-D2pMxC2I.js +1 -0
- package/static/app/assets/Library-DwO3yZST.js +1 -0
- package/static/app/assets/{LivingBrain-BXMWIK_2.js → LivingBrain-Jn1GK0-S.js} +1 -1
- package/static/app/assets/ProductFlow-B-w1R4Oo.js +1 -0
- package/static/app/assets/ReviewCard-6B27X8Vg.js +3 -0
- package/static/app/assets/System-DW8F-2xL.js +1 -0
- package/static/app/assets/arrow-left-DXvKg9U6.js +1 -0
- package/static/app/assets/{bot-4BvN07ux.js → bot-IM_E_Y12.js} +1 -1
- package/static/app/assets/brain-Ci1CkWjM.js +1 -0
- package/static/app/assets/{button-CDjtnAoU.js → button-COwyqfHM.js} +1 -1
- package/static/app/assets/circle-check-DfInj-qD.js +1 -0
- package/static/app/assets/{circle-pause-D_RMn7tp.js → circle-pause-DEM4A1Y5.js} +1 -1
- package/static/app/assets/{circle-play-B5OpB8ae.js → circle-play-C9djDuLd.js} +1 -1
- package/static/app/assets/{cpu-BIlWInHf.js → cpu-DFdo1gw-.js} +1 -1
- package/static/app/assets/{download-BtjXfL3z.js → download-SnJL6oqk.js} +1 -1
- package/static/app/assets/{folder-open-DefMpxI2.js → folder-open-CqZeDkjE.js} +1 -1
- package/static/app/assets/{hard-drive-BQ8NZVkw.js → hard-drive-j1jJXYYf.js} +1 -1
- package/static/app/assets/{index-vtEfYvQY.css → index-BLPb5lmE.css} +1 -1
- package/static/app/assets/index-_u5iUHDr.js +10 -0
- package/static/app/assets/input-B0lPdRQZ.js +1 -0
- package/static/app/assets/link-2-CoFbooHS.js +1 -0
- package/static/app/assets/{permissionCopy-BqZ5tsgL.js → permissionCopy-BsyLxtao.js} +1 -1
- package/static/app/assets/primitives-DEbN-d6p.js +1 -0
- package/static/app/assets/search-BybIWPNd.js +1 -0
- package/static/app/assets/{share-2-D5zg_0fY.js → share-2-CVtZ_ewX.js} +1 -1
- package/static/app/assets/{shield-alert-B5pZzkUb.js → shield-alert-CBi2GNWM.js} +1 -1
- package/static/app/assets/{textarea-nEVIweKY.js → textarea-DNMpB5ih.js} +1 -1
- package/static/app/assets/{useFocusTrap-Cm99AHlz.js → useFocusTrap-C83t3GXF.js} +1 -1
- package/static/app/assets/useMutation-DtbJDoyz.js +1 -0
- package/static/app/assets/{useQuery-Dm__N6bL.js → useQuery-Dcp1OChy.js} +1 -1
- package/static/app/assets/utils-BlZr7Pd4.js +4 -0
- package/static/app/assets/workspace-jJY4RuAV.js +1 -0
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/lattice_brain/graph/_kg_common.py +0 -1331
- package/lattice_brain/graph/discovery_index.py +0 -1141
- package/lattice_brain/graph/retrieval.py +0 -1120
- package/lattice_brain/graph/retrieval_vector.py +0 -1293
- package/lattice_brain/ingestion.py +0 -1377
- package/lattice_brain/portability.py +0 -1210
- package/latticeai/core/agent.py +0 -1412
- package/latticeai/core/embedding_providers.py +0 -1196
- package/latticeai/core/file_generation.py +0 -1047
- package/latticeai/integrations/telegram_bot.py +0 -1390
- package/latticeai/models/router.py +0 -1007
- package/latticeai/runtime/build_phases.py +0 -1422
- package/latticeai/services/brain_intelligence.py +0 -967
- package/latticeai/services/memory_service.py +0 -1177
- package/latticeai/services/model_runtime.py +0 -1281
- package/latticeai/setup/wizard.py +0 -1277
- package/static/app/assets/Act-D0HWqtn0.js +0 -1
- package/static/app/assets/AdminConsole-D-QDW-A4.js +0 -1
- package/static/app/assets/BrainHome-Btns-_TA.js +0 -2
- package/static/app/assets/BrainSignals-2dHQNkns.js +0 -1
- package/static/app/assets/Capture-CT8v1StE.js +0 -1
- package/static/app/assets/CommandPalette-DoLXC2KH.js +0 -1
- package/static/app/assets/Library-DDoxFE5c.js +0 -1
- package/static/app/assets/ProductFlow-DOYf7JIs.js +0 -1
- package/static/app/assets/ReviewCard-COQsqidK.js +0 -3
- package/static/app/assets/System-BRllvYXd.js +0 -1
- package/static/app/assets/arrow-left-DnyMzss-.js +0 -1
- package/static/app/assets/brain-uMb_5hnO.js +0 -1
- package/static/app/assets/index-0AvoEBzJ.js +0 -10
- package/static/app/assets/input-B_5ZJ9oy.js +0 -1
- package/static/app/assets/primitives-CVwew78r.js +0 -1
- package/static/app/assets/search-DkhnOKZt.js +0 -1
- package/static/app/assets/utils-DcDMoZIe.js +0 -4
- package/static/app/assets/workspace-LtRRSKTf.js +0 -1
|
@@ -1,39 +1,32 @@
|
|
|
1
|
+
"""The legacy → v2 projection: schema, backfill, per-row writes, mirrors.
|
|
2
|
+
|
|
3
|
+
The v2 tables and the ``kgv2_*`` reconstruction views are *derived* — the legacy
|
|
4
|
+
``nodes``/``edges`` tables stay authoritative — which is what makes the whole
|
|
5
|
+
DROP → CREATE → VIEWS → BACKFILL → stamp migration safe to run in one
|
|
6
|
+
transaction and simply retry on the next startup when it fails. The trigram FTS
|
|
7
|
+
index lives here too: it is the other index projected off the same rows.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
# ruff: noqa: F403,F405,S608
|
|
1
11
|
from __future__ import annotations
|
|
2
12
|
|
|
3
13
|
from typing import TYPE_CHECKING
|
|
4
14
|
|
|
5
|
-
from
|
|
6
|
-
|
|
7
|
-
# ruff: noqa: F403,F405
|
|
8
|
-
from ._kg_common import * # noqa: F403,F401
|
|
15
|
+
from ...quiet import quiet
|
|
16
|
+
from .._kg_common import * # noqa: F401
|
|
9
17
|
|
|
10
18
|
# The cross-mixin surface (`_connect`, `_upsert_node`, …) is declared in
|
|
11
19
|
# `_kg_contract.KnowledgeGraphCore`. It is a typing-only base: at runtime this
|
|
12
20
|
# is `object`, so the MRO of `KnowledgeGraphStore` is unchanged.
|
|
13
21
|
if TYPE_CHECKING:
|
|
14
|
-
from
|
|
22
|
+
from .._kg_contract import KnowledgeGraphCore as _Core
|
|
15
23
|
else:
|
|
16
24
|
_Core = object
|
|
17
25
|
|
|
18
26
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# human decision instead of writing them immediately. Explicit review_mode=
|
|
22
|
-
# argument wins; otherwise this env opt-in decides; default stays auto-promote.
|
|
23
|
-
_PROMOTION_REVIEW_ENV = "LATTICEAI_GRAPH_PROMOTION_REVIEW"
|
|
24
|
-
_PENDING_PROMOTIONS_KEY = "pending_promotions"
|
|
25
|
-
_PENDING_PROMOTIONS_CAP = 100
|
|
26
|
-
|
|
27
|
-
# graph_meta stamp written by an applied (dry_run=False) noise-curate run; the
|
|
28
|
-
# Command Center hygiene advisory reads it to pace its suggestion (Wave 2.5).
|
|
29
|
-
_LAST_NOISE_CURATE_KEY = "last_noise_curate_at"
|
|
27
|
+
class KnowledgeGraphV2SchemaMixin(_Core):
|
|
28
|
+
"""Projection + FTS. Mixed into ``KnowledgeGraphProjectionMixin``."""
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
def _promotion_review_default() -> bool:
|
|
33
|
-
return os.getenv(_PROMOTION_REVIEW_ENV, "").strip().lower() in ("1", "true", "yes")
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class KnowledgeGraphProjectionMixin(_Core):
|
|
37
30
|
_FTS_SQL = """
|
|
38
31
|
CREATE VIRTUAL TABLE IF NOT EXISTS node_fts USING fts5(
|
|
39
32
|
node_id UNINDEXED, title, summary, metadata, tokenize='trigram'
|
|
@@ -52,16 +45,29 @@ class KnowledgeGraphProjectionMixin(_Core):
|
|
|
52
45
|
END;
|
|
53
46
|
"""
|
|
54
47
|
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
#
|
|
48
|
+
# ``type`` is reconstructed from ``legacy_type`` when there is one, because
|
|
49
|
+
# that column carries the label a reader expects. The trap the 11.0.1
|
|
50
|
+
# review recorded — and which 11.2.0 fixes — is that ``edges_v2.legacy_type``
|
|
51
|
+
# is ``NOT NULL DEFAULT ''``: a natively-canonical edge is written with
|
|
52
|
+
# ``legacy_type=''`` so its identity is effectively (source, target, type),
|
|
53
|
+
# and ``COALESCE('', type)`` returns ``''`` because COALESCE only skips
|
|
54
|
+
# NULL. Every canonical edge therefore read back with an **empty type**.
|
|
55
|
+
# ``NULLIF(legacy_type, '')`` is the fix: empty means "no legacy label",
|
|
56
|
+
# which is precisely what the write side meant by it.
|
|
57
|
+
#
|
|
58
|
+
# The empty string stays the write-side sentinel on purpose. SQLite treats
|
|
59
|
+
# NULLs as distinct in a UNIQUE index, so moving to NULL would silently
|
|
60
|
+
# disable the (source, target, type, legacy_type) dedupe and let the same
|
|
61
|
+
# relation land twice.
|
|
62
|
+
#
|
|
63
|
+
# The temporal columns pass through *raw* (v11.1.0): validity is not a
|
|
64
|
+
# label, and a COALESCE there would turn "still valid" (NULL) into a value.
|
|
65
|
+
# NULL in, NULL out; the fallback to ``created_at`` belongs to the read
|
|
66
|
+
# predicate (``schema.TEMPORAL_PREDICATE_SQL``), not to the view.
|
|
61
67
|
_V2_VIEWS_SQL = """
|
|
62
68
|
CREATE VIEW IF NOT EXISTS kgv2_nodes AS
|
|
63
69
|
SELECT id,
|
|
64
|
-
COALESCE(legacy_type, type) AS type,
|
|
70
|
+
COALESCE(NULLIF(legacy_type, ''), type) AS type,
|
|
65
71
|
label AS title,
|
|
66
72
|
summary,
|
|
67
73
|
attrs AS metadata_json,
|
|
@@ -70,7 +76,7 @@ class KnowledgeGraphProjectionMixin(_Core):
|
|
|
70
76
|
FROM nodes_v2;
|
|
71
77
|
CREATE VIEW IF NOT EXISTS kgv2_edges AS
|
|
72
78
|
SELECT id, source AS from_node, target AS to_node,
|
|
73
|
-
COALESCE(legacy_type, type) AS type,
|
|
79
|
+
COALESCE(NULLIF(legacy_type, ''), type) AS type,
|
|
74
80
|
weight,
|
|
75
81
|
metadata AS metadata_json,
|
|
76
82
|
created_at,
|
|
@@ -156,6 +162,7 @@ class KnowledgeGraphProjectionMixin(_Core):
|
|
|
156
162
|
KGStoreV2(self.db_path).init_schema(conn=conn)
|
|
157
163
|
_exec_script(conn, self._V2_VIEWS_SQL)
|
|
158
164
|
self._backfill_v2_on(conn, force=stale)
|
|
165
|
+
self._normalize_v2_legacy_types(conn)
|
|
159
166
|
# version stamp commits together with the backfill — never stranded
|
|
160
167
|
conn.execute(
|
|
161
168
|
"INSERT OR REPLACE INTO kg_meta(key, value) VALUES ('projection_version', ?)",
|
|
@@ -177,6 +184,50 @@ class KnowledgeGraphProjectionMixin(_Core):
|
|
|
177
184
|
except Exception as e:
|
|
178
185
|
logging.warning("knowledge_graph: v2 schema init/backfill skipped: %s", e)
|
|
179
186
|
|
|
187
|
+
def _normalize_v2_legacy_types(self, conn: sqlite3.Connection) -> Dict[str, int]:
|
|
188
|
+
"""Bring pre-11.2.0 rows onto the current ``legacy_type`` convention.
|
|
189
|
+
|
|
190
|
+
The convention is: ``legacy_type`` holds the *raw* label only when it
|
|
191
|
+
differs from the canonical type, and ``''`` (edges) / ``NULL`` (nodes)
|
|
192
|
+
otherwise. Rows written by older builds could carry the canonical value
|
|
193
|
+
in both columns, which is redundant and — because the dedupe key
|
|
194
|
+
includes ``legacy_type`` — splits one relation across two rows.
|
|
195
|
+
|
|
196
|
+
Idempotent by construction: a second run matches nothing. ``UPDATE OR
|
|
197
|
+
IGNORE`` is used because collapsing a redundant row can collide with
|
|
198
|
+
the canonical one that already exists; those survivors are counted and
|
|
199
|
+
reported rather than deleted, since the fixed view reads both
|
|
200
|
+
identically and deleting an edge is not a migration's business.
|
|
201
|
+
"""
|
|
202
|
+
report = {"edges": 0, "nodes": 0, "collisions": 0}
|
|
203
|
+
try:
|
|
204
|
+
before = int(
|
|
205
|
+
conn.execute(
|
|
206
|
+
"SELECT COUNT(*) FROM edges_v2 WHERE legacy_type = type"
|
|
207
|
+
).fetchone()[0]
|
|
208
|
+
or 0
|
|
209
|
+
)
|
|
210
|
+
conn.execute(
|
|
211
|
+
"UPDATE OR IGNORE edges_v2 SET legacy_type='' WHERE legacy_type = type"
|
|
212
|
+
)
|
|
213
|
+
after = int(
|
|
214
|
+
conn.execute(
|
|
215
|
+
"SELECT COUNT(*) FROM edges_v2 WHERE legacy_type = type"
|
|
216
|
+
).fetchone()[0]
|
|
217
|
+
or 0
|
|
218
|
+
)
|
|
219
|
+
report["edges"] = before - after
|
|
220
|
+
report["collisions"] = after
|
|
221
|
+
cursor = conn.execute(
|
|
222
|
+
"UPDATE nodes_v2 SET legacy_type=NULL WHERE legacy_type = ''"
|
|
223
|
+
)
|
|
224
|
+
report["nodes"] = int(cursor.rowcount or 0)
|
|
225
|
+
except sqlite3.Error as exc:
|
|
226
|
+
# The projection is derived; a migration that cannot run leaves the
|
|
227
|
+
# data exactly as it was and the fixed view still reads it right.
|
|
228
|
+
logging.debug("knowledge_graph: legacy_type normalization skipped: %s", exc)
|
|
229
|
+
return report
|
|
230
|
+
|
|
180
231
|
def _backup_before_v2_flip(self) -> Optional[str]:
|
|
181
232
|
"""Create one local SQLite backup before the v2 write-master flip."""
|
|
182
233
|
if not self.db_path.exists() or self.db_path.stat().st_size == 0:
|
|
@@ -443,461 +494,6 @@ class KnowledgeGraphProjectionMixin(_Core):
|
|
|
443
494
|
ex,
|
|
444
495
|
)
|
|
445
496
|
|
|
446
|
-
def curate(
|
|
447
|
-
self,
|
|
448
|
-
*,
|
|
449
|
-
max_documents: int = 200,
|
|
450
|
-
max_new_nodes: int = 8,
|
|
451
|
-
review_mode: Optional[bool] = None,
|
|
452
|
-
) -> Dict[str, Any]:
|
|
453
|
-
"""On-demand graph curation (T4.4 — graph_curator goes live).
|
|
454
|
-
|
|
455
|
-
Runs the curator's gated topic-promotion pipeline over recent content
|
|
456
|
-
nodes: candidates are clustered, secret-bearing labels are refused,
|
|
457
|
-
and only multi-source topics above the importance threshold become
|
|
458
|
-
Topic nodes (with MENTIONS edges back to their sources and a real
|
|
459
|
-
importance_score in nodes_v2). Explicit and observable — the result
|
|
460
|
-
reports everything promoted AND everything skipped, with reasons.
|
|
461
|
-
|
|
462
|
-
``review_mode`` (review 2026-07-25 Wave 4): when True, nothing is
|
|
463
|
-
written — the would-be promotions are parked in ``graph_meta`` as
|
|
464
|
-
``pending_promotions`` for a human decision via
|
|
465
|
-
:meth:`apply_pending_promotions` / :meth:`reject_pending_promotions`.
|
|
466
|
-
Explicit argument wins; ``None`` falls back to the
|
|
467
|
-
``LATTICEAI_GRAPH_PROMOTION_REVIEW`` env opt-in; default stays the
|
|
468
|
-
historical auto-promote behavior.
|
|
469
|
-
"""
|
|
470
|
-
from .curator import auto_build_graph_overlay
|
|
471
|
-
|
|
472
|
-
content_types = (
|
|
473
|
-
"Document",
|
|
474
|
-
"File",
|
|
475
|
-
"CodeFile",
|
|
476
|
-
"Message",
|
|
477
|
-
"AIResponse",
|
|
478
|
-
"Chat",
|
|
479
|
-
"Page",
|
|
480
|
-
"Slide",
|
|
481
|
-
"Spreadsheet",
|
|
482
|
-
)
|
|
483
|
-
nt, _ = self._read_tables()
|
|
484
|
-
with self._connect() as conn:
|
|
485
|
-
placeholders = ",".join("?" for _ in content_types)
|
|
486
|
-
rows = conn.execute(
|
|
487
|
-
f"""
|
|
488
|
-
SELECT id, type, title, summary FROM {nt}
|
|
489
|
-
WHERE type IN ({placeholders})
|
|
490
|
-
ORDER BY updated_at DESC, id ASC LIMIT ?
|
|
491
|
-
""",
|
|
492
|
-
(*content_types, max(1, min(int(max_documents), 2000))),
|
|
493
|
-
).fetchall()
|
|
494
|
-
existing_labels = {
|
|
495
|
-
str(row["title"] or "").strip().lower()
|
|
496
|
-
for row in conn.execute(
|
|
497
|
-
f"SELECT title FROM {nt} WHERE type IN ('Topic', 'Concept')"
|
|
498
|
-
).fetchall()
|
|
499
|
-
}
|
|
500
|
-
documents = [
|
|
501
|
-
{
|
|
502
|
-
"id": row["id"],
|
|
503
|
-
"text": f"{row['title']} {row['summary'] or ''}",
|
|
504
|
-
"kind": "file"
|
|
505
|
-
if row["type"] in {"Document", "File", "CodeFile", "Spreadsheet"}
|
|
506
|
-
else "chat",
|
|
507
|
-
}
|
|
508
|
-
for row in rows
|
|
509
|
-
]
|
|
510
|
-
overlay = auto_build_graph_overlay(
|
|
511
|
-
documents,
|
|
512
|
-
existing_node_labels=existing_labels,
|
|
513
|
-
max_new_nodes=max(1, min(int(max_new_nodes), 50)),
|
|
514
|
-
)
|
|
515
|
-
valid_ids = {row["id"] for row in rows}
|
|
516
|
-
review = review_mode if review_mode is not None else _promotion_review_default()
|
|
517
|
-
if review:
|
|
518
|
-
proposed_at = _now()
|
|
519
|
-
proposed = [
|
|
520
|
-
{
|
|
521
|
-
"id": f"topic:{_slug(promo['label'])}",
|
|
522
|
-
"label": promo["label"],
|
|
523
|
-
"importance": promo["importance"],
|
|
524
|
-
"aliases": promo["aliases"],
|
|
525
|
-
"sources": [s for s in promo["sources"][:10] if s in valid_ids],
|
|
526
|
-
"proposed_at": proposed_at,
|
|
527
|
-
}
|
|
528
|
-
for promo in overlay["promotions"]
|
|
529
|
-
]
|
|
530
|
-
with self._connect() as conn:
|
|
531
|
-
merged = self._merge_pending_promotions(conn, proposed)
|
|
532
|
-
return {
|
|
533
|
-
"status": "pending_review",
|
|
534
|
-
"documents_scanned": len(documents),
|
|
535
|
-
"candidates_total": overlay["candidates_total"],
|
|
536
|
-
"pending": proposed,
|
|
537
|
-
"pending_total": len(merged),
|
|
538
|
-
"skipped": overlay["skipped"][:50],
|
|
539
|
-
"skipped_total": len(overlay["skipped"]),
|
|
540
|
-
}
|
|
541
|
-
promoted: List[Dict[str, Any]] = []
|
|
542
|
-
with self._connect() as conn:
|
|
543
|
-
for promo in overlay["promotions"]:
|
|
544
|
-
promoted.append(
|
|
545
|
-
self._write_promotion(conn, promo, valid_source_ids=valid_ids)
|
|
546
|
-
)
|
|
547
|
-
return {
|
|
548
|
-
"status": "ok",
|
|
549
|
-
"documents_scanned": len(documents),
|
|
550
|
-
"candidates_total": overlay["candidates_total"],
|
|
551
|
-
"promoted": promoted,
|
|
552
|
-
"skipped": overlay["skipped"][:50],
|
|
553
|
-
"skipped_total": len(overlay["skipped"]),
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
def _write_promotion(
|
|
557
|
-
self,
|
|
558
|
-
conn: sqlite3.Connection,
|
|
559
|
-
promo: Dict[str, Any],
|
|
560
|
-
*,
|
|
561
|
-
valid_source_ids: Optional[set] = None,
|
|
562
|
-
) -> Dict[str, Any]:
|
|
563
|
-
"""Write one curator promotion: Topic node + importance + MENTIONS edges.
|
|
564
|
-
|
|
565
|
-
Single write path shared by direct ``curate()`` and
|
|
566
|
-
:meth:`apply_pending_promotions`, so a human-approved promotion lands
|
|
567
|
-
exactly like an auto-promoted one. ``valid_source_ids`` restricts the
|
|
568
|
-
linkable sources to this curate run's scanned rows; when ``None``
|
|
569
|
-
(apply-after-review), each stored source is checked for existence so a
|
|
570
|
-
node deleted between propose and apply is skipped, not an error.
|
|
571
|
-
"""
|
|
572
|
-
topic_id = str(promo.get("id") or f"topic:{_slug(str(promo['label']))}")
|
|
573
|
-
self._upsert_node(
|
|
574
|
-
conn,
|
|
575
|
-
topic_id,
|
|
576
|
-
"Topic",
|
|
577
|
-
str(promo["label"]),
|
|
578
|
-
metadata={
|
|
579
|
-
"curated": True,
|
|
580
|
-
"importance": promo["importance"],
|
|
581
|
-
"aliases": list(promo.get("aliases") or []),
|
|
582
|
-
"source": "graph_curator",
|
|
583
|
-
},
|
|
584
|
-
)
|
|
585
|
-
conn.execute(
|
|
586
|
-
"UPDATE nodes_v2 SET importance_score=? WHERE id=?",
|
|
587
|
-
(float(promo["importance"]), topic_id),
|
|
588
|
-
)
|
|
589
|
-
linked = 0
|
|
590
|
-
for source_id in list(promo.get("sources") or [])[:10]:
|
|
591
|
-
if valid_source_ids is not None:
|
|
592
|
-
if source_id not in valid_source_ids:
|
|
593
|
-
continue
|
|
594
|
-
elif not conn.execute(
|
|
595
|
-
"SELECT 1 FROM nodes WHERE id=?", (source_id,)
|
|
596
|
-
).fetchone():
|
|
597
|
-
continue
|
|
598
|
-
self._upsert_edge(
|
|
599
|
-
conn,
|
|
600
|
-
source_id,
|
|
601
|
-
topic_id,
|
|
602
|
-
"MENTIONS",
|
|
603
|
-
weight=0.6,
|
|
604
|
-
metadata={"source": "graph_curator"},
|
|
605
|
-
)
|
|
606
|
-
linked += 1
|
|
607
|
-
return {
|
|
608
|
-
"node_id": topic_id,
|
|
609
|
-
"label": promo["label"],
|
|
610
|
-
"importance": promo["importance"],
|
|
611
|
-
"linked_sources": linked,
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
# ── pending promotion queue (review 2026-07-25 Wave 4) ───────────────────
|
|
615
|
-
|
|
616
|
-
def _read_pending_promotions(
|
|
617
|
-
self, conn: sqlite3.Connection
|
|
618
|
-
) -> List[Dict[str, Any]]:
|
|
619
|
-
try:
|
|
620
|
-
row = conn.execute(
|
|
621
|
-
"SELECT value FROM graph_meta WHERE key=?",
|
|
622
|
-
(_PENDING_PROMOTIONS_KEY,),
|
|
623
|
-
).fetchone()
|
|
624
|
-
except sqlite3.Error:
|
|
625
|
-
return []
|
|
626
|
-
if not row or not row["value"]:
|
|
627
|
-
return []
|
|
628
|
-
try:
|
|
629
|
-
parsed = json.loads(row["value"])
|
|
630
|
-
except (TypeError, ValueError):
|
|
631
|
-
return []
|
|
632
|
-
if not isinstance(parsed, list):
|
|
633
|
-
return []
|
|
634
|
-
return [
|
|
635
|
-
item for item in parsed if isinstance(item, dict) and item.get("id")
|
|
636
|
-
]
|
|
637
|
-
|
|
638
|
-
def _store_pending_promotions(
|
|
639
|
-
self, conn: sqlite3.Connection, entries: List[Dict[str, Any]]
|
|
640
|
-
) -> None:
|
|
641
|
-
conn.execute(
|
|
642
|
-
"INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
|
|
643
|
-
(_PENDING_PROMOTIONS_KEY, json.dumps(entries, ensure_ascii=False)),
|
|
644
|
-
)
|
|
645
|
-
|
|
646
|
-
def _merge_pending_promotions(
|
|
647
|
-
self, conn: sqlite3.Connection, proposed: List[Dict[str, Any]]
|
|
648
|
-
) -> List[Dict[str, Any]]:
|
|
649
|
-
"""Merge new proposals into the stored queue (dedupe by id, cap 100)."""
|
|
650
|
-
merged: Dict[str, Dict[str, Any]] = {}
|
|
651
|
-
for item in self._read_pending_promotions(conn) + list(proposed):
|
|
652
|
-
merged[str(item["id"])] = item # newest proposal wins per id
|
|
653
|
-
entries = list(merged.values())[-_PENDING_PROMOTIONS_CAP:]
|
|
654
|
-
self._store_pending_promotions(conn, entries)
|
|
655
|
-
return entries
|
|
656
|
-
|
|
657
|
-
def pending_promotions(self) -> List[Dict[str, Any]]:
|
|
658
|
-
"""List promotions waiting for a human decision (review mode)."""
|
|
659
|
-
with self._connect() as conn:
|
|
660
|
-
return self._read_pending_promotions(conn)
|
|
661
|
-
|
|
662
|
-
def apply_pending_promotions(
|
|
663
|
-
self, ids: Optional[List[str]] = None
|
|
664
|
-
) -> Dict[str, Any]:
|
|
665
|
-
"""Apply stored pending promotions (all of them when ``ids`` is None).
|
|
666
|
-
|
|
667
|
-
Uses the exact node-writing path as direct ``curate()`` via
|
|
668
|
-
:meth:`_write_promotion`; applied entries leave the queue.
|
|
669
|
-
"""
|
|
670
|
-
wanted = None if ids is None else {str(item) for item in ids}
|
|
671
|
-
applied: List[Dict[str, Any]] = []
|
|
672
|
-
remaining: List[Dict[str, Any]] = []
|
|
673
|
-
with self._connect() as conn:
|
|
674
|
-
for promo in self._read_pending_promotions(conn):
|
|
675
|
-
if wanted is not None and str(promo.get("id")) not in wanted:
|
|
676
|
-
remaining.append(promo)
|
|
677
|
-
continue
|
|
678
|
-
applied.append(self._write_promotion(conn, promo))
|
|
679
|
-
self._store_pending_promotions(conn, remaining)
|
|
680
|
-
return {"status": "ok", "applied": applied, "remaining": len(remaining)}
|
|
681
|
-
|
|
682
|
-
def reject_pending_promotions(
|
|
683
|
-
self, ids: Optional[List[str]] = None
|
|
684
|
-
) -> Dict[str, Any]:
|
|
685
|
-
"""Drop pending promotions without writing (all when ``ids`` is None)."""
|
|
686
|
-
wanted = None if ids is None else {str(item) for item in ids}
|
|
687
|
-
rejected: List[str] = []
|
|
688
|
-
remaining: List[Dict[str, Any]] = []
|
|
689
|
-
with self._connect() as conn:
|
|
690
|
-
for promo in self._read_pending_promotions(conn):
|
|
691
|
-
if wanted is not None and str(promo.get("id")) not in wanted:
|
|
692
|
-
remaining.append(promo)
|
|
693
|
-
continue
|
|
694
|
-
rejected.append(str(promo.get("id")))
|
|
695
|
-
self._store_pending_promotions(conn, remaining)
|
|
696
|
-
return {"status": "ok", "rejected": rejected, "remaining": len(remaining)}
|
|
697
|
-
|
|
698
|
-
_NOISE_CONTENT_TYPES = (
|
|
699
|
-
"Document",
|
|
700
|
-
"File",
|
|
701
|
-
"CodeFile",
|
|
702
|
-
"Message",
|
|
703
|
-
"AIResponse",
|
|
704
|
-
"Chat",
|
|
705
|
-
"Page",
|
|
706
|
-
"Slide",
|
|
707
|
-
"Spreadsheet",
|
|
708
|
-
)
|
|
709
|
-
_NOISE_CONCEPT_TYPES = ("Concept", "Feature", "Topic", "Code", "Error")
|
|
710
|
-
|
|
711
|
-
def curate_noise(
|
|
712
|
-
self,
|
|
713
|
-
*,
|
|
714
|
-
dry_run: bool = True,
|
|
715
|
-
max_df_ratio: float = 0.8,
|
|
716
|
-
min_doc_frequency: int = 1,
|
|
717
|
-
min_corpus_docs: int = 5,
|
|
718
|
-
normalize_verbs: bool = True,
|
|
719
|
-
max_removals: int = 200,
|
|
720
|
-
) -> Dict[str, Any]:
|
|
721
|
-
"""Noise-reduction curation job (backlog #10, review §7.2 D).
|
|
722
|
-
|
|
723
|
-
(a) Removes heuristic concept nodes (``auto_extracted`` /
|
|
724
|
-
``graph_curator``-promoted) whose document frequency marks them as
|
|
725
|
-
noise: ubiquitous (low IDF — linked from more than ``max_df_ratio`` of
|
|
726
|
-
content docs once the corpus has ``min_corpus_docs``) or below the
|
|
727
|
-
``min_doc_frequency`` floor. Explicitly user-created nodes are never
|
|
728
|
-
touched, whatever their stats.
|
|
729
|
-
|
|
730
|
-
(b) Normalizes free-string relation verbs on the legacy edge table via
|
|
731
|
-
the ko/en dictionary in :mod:`lattice_brain.graph.curator`
|
|
732
|
-
('만들다/만든/creates' → 'created', …), merging rows that collide
|
|
733
|
-
after the rename.
|
|
734
|
-
|
|
735
|
-
``dry_run=True`` (the default) only *reports* what would change.
|
|
736
|
-
"""
|
|
737
|
-
from .curator import (
|
|
738
|
-
build_relation_verb_index,
|
|
739
|
-
plan_concept_noise_reduction,
|
|
740
|
-
plan_relation_normalization,
|
|
741
|
-
)
|
|
742
|
-
|
|
743
|
-
max_removals = max(0, int(max_removals))
|
|
744
|
-
# Operate on the legacy write tables directly: they are the mutation
|
|
745
|
-
# target, and raw free-string verbs only exist there (the v4 write
|
|
746
|
-
# door normalizes new edges; the kgv2_* read views collapse
|
|
747
|
-
# legacy_type and would hide exactly the rows this job cleans up).
|
|
748
|
-
nt, et = "nodes", "edges"
|
|
749
|
-
with self._connect() as conn:
|
|
750
|
-
content_ph = ",".join("?" for _ in self._NOISE_CONTENT_TYPES)
|
|
751
|
-
total_docs = conn.execute(
|
|
752
|
-
f"SELECT COUNT(*) AS c FROM {nt} WHERE type IN ({content_ph})",
|
|
753
|
-
self._NOISE_CONTENT_TYPES,
|
|
754
|
-
).fetchone()["c"]
|
|
755
|
-
|
|
756
|
-
concept_ph = ",".join("?" for _ in self._NOISE_CONCEPT_TYPES)
|
|
757
|
-
concept_rows = conn.execute(
|
|
758
|
-
f"SELECT id, type, title, metadata_json FROM {nt} WHERE type IN ({concept_ph})",
|
|
759
|
-
self._NOISE_CONCEPT_TYPES,
|
|
760
|
-
).fetchall()
|
|
761
|
-
concepts = []
|
|
762
|
-
for row in concept_rows:
|
|
763
|
-
meta = _safe_loads(row["metadata_json"]) or {}
|
|
764
|
-
heuristic = bool(meta.get("auto_extracted")) or (
|
|
765
|
-
meta.get("source") == "graph_curator" or meta.get("curated") is True
|
|
766
|
-
)
|
|
767
|
-
# Document frequency: distinct *content* nodes linked to this
|
|
768
|
-
# concept in either direction.
|
|
769
|
-
df = conn.execute(
|
|
770
|
-
f"""
|
|
771
|
-
SELECT COUNT(DISTINCT n.id) AS c
|
|
772
|
-
FROM {et} e
|
|
773
|
-
JOIN {nt} n
|
|
774
|
-
ON n.id = CASE WHEN e.to_node = ? THEN e.from_node ELSE e.to_node END
|
|
775
|
-
WHERE (e.to_node = ? OR e.from_node = ?)
|
|
776
|
-
AND n.type IN ({content_ph})
|
|
777
|
-
""",
|
|
778
|
-
(row["id"], row["id"], row["id"], *self._NOISE_CONTENT_TYPES),
|
|
779
|
-
).fetchone()["c"]
|
|
780
|
-
concepts.append({
|
|
781
|
-
"id": row["id"],
|
|
782
|
-
"label": row["title"],
|
|
783
|
-
"type": row["type"],
|
|
784
|
-
"df": int(df or 0),
|
|
785
|
-
"heuristic": heuristic,
|
|
786
|
-
})
|
|
787
|
-
|
|
788
|
-
plan = plan_concept_noise_reduction(
|
|
789
|
-
concepts,
|
|
790
|
-
total_docs,
|
|
791
|
-
max_df_ratio=max_df_ratio,
|
|
792
|
-
min_doc_frequency=min_doc_frequency,
|
|
793
|
-
min_corpus_docs=min_corpus_docs,
|
|
794
|
-
)
|
|
795
|
-
removals = plan["remove"][:max_removals]
|
|
796
|
-
|
|
797
|
-
verb_index = build_relation_verb_index()
|
|
798
|
-
edge_type_rows = conn.execute(
|
|
799
|
-
f"SELECT DISTINCT type FROM {et}"
|
|
800
|
-
).fetchall()
|
|
801
|
-
verb_plan = (
|
|
802
|
-
plan_relation_normalization(
|
|
803
|
-
(row["type"] for row in edge_type_rows), index=verb_index,
|
|
804
|
-
)
|
|
805
|
-
if normalize_verbs
|
|
806
|
-
else {}
|
|
807
|
-
)
|
|
808
|
-
|
|
809
|
-
removed_count = 0
|
|
810
|
-
renamed_edges = 0
|
|
811
|
-
if not dry_run:
|
|
812
|
-
for decision in removals:
|
|
813
|
-
node_id = decision["id"]
|
|
814
|
-
conn.execute(
|
|
815
|
-
"DELETE FROM edges WHERE from_node=? OR to_node=?",
|
|
816
|
-
(node_id, node_id),
|
|
817
|
-
)
|
|
818
|
-
conn.execute(
|
|
819
|
-
"DELETE FROM vector_embeddings WHERE item_id=?", (node_id,)
|
|
820
|
-
)
|
|
821
|
-
conn.execute("DELETE FROM nodes WHERE id=?", (node_id,))
|
|
822
|
-
self._v2_delete_nodes(conn, [node_id])
|
|
823
|
-
removed_count += 1
|
|
824
|
-
for original, canonical in verb_plan.items():
|
|
825
|
-
renamed_edges += conn.execute(
|
|
826
|
-
"SELECT COUNT(*) AS c FROM edges WHERE type=?", (original,)
|
|
827
|
-
).fetchone()["c"]
|
|
828
|
-
# UNIQUE(from_node, to_node, type): merge rows that collide
|
|
829
|
-
# after the rename instead of failing the UPDATE.
|
|
830
|
-
conn.execute(
|
|
831
|
-
"UPDATE OR IGNORE edges SET type=? WHERE type=?",
|
|
832
|
-
(canonical, original),
|
|
833
|
-
)
|
|
834
|
-
conn.execute("DELETE FROM edges WHERE type=?", (original,))
|
|
835
|
-
# Stamp every applied run — even a no-op one means the graph
|
|
836
|
-
# was inspected, so the Command Center hygiene advisory
|
|
837
|
-
# (review 2026-07-25 Wave 2.5) stops re-suggesting for a while.
|
|
838
|
-
conn.execute(
|
|
839
|
-
"INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
|
|
840
|
-
(_LAST_NOISE_CURATE_KEY, _now()),
|
|
841
|
-
)
|
|
842
|
-
|
|
843
|
-
return {
|
|
844
|
-
"status": "ok",
|
|
845
|
-
"dry_run": bool(dry_run),
|
|
846
|
-
"total_content_docs": int(total_docs or 0),
|
|
847
|
-
"concepts_examined": len(concepts),
|
|
848
|
-
"remove": removals,
|
|
849
|
-
"remove_total": len(plan["remove"]),
|
|
850
|
-
"kept": len(plan["keep"]),
|
|
851
|
-
"protected_user_nodes": sum(
|
|
852
|
-
1 for item in plan["keep"] if item.get("reason") == "user_created_protected"
|
|
853
|
-
),
|
|
854
|
-
"verb_normalizations": verb_plan,
|
|
855
|
-
"applied": {
|
|
856
|
-
"removed_nodes": removed_count,
|
|
857
|
-
"renamed_edges": renamed_edges,
|
|
858
|
-
},
|
|
859
|
-
"thresholds": {
|
|
860
|
-
"max_df_ratio": float(max_df_ratio),
|
|
861
|
-
"min_doc_frequency": int(min_doc_frequency),
|
|
862
|
-
"min_corpus_docs": int(min_corpus_docs),
|
|
863
|
-
},
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
def last_noise_curate_at(self) -> Optional[str]:
|
|
867
|
-
"""Timestamp of the last applied (dry_run=False) noise-curate run.
|
|
868
|
-
|
|
869
|
-
``None`` when the job never ran or the meta table is unreadable —
|
|
870
|
-
advisory readers treat both as "curation is due" (fail-open).
|
|
871
|
-
"""
|
|
872
|
-
try:
|
|
873
|
-
with self._connect() as conn:
|
|
874
|
-
row = conn.execute(
|
|
875
|
-
"SELECT value FROM graph_meta WHERE key=?",
|
|
876
|
-
(_LAST_NOISE_CURATE_KEY,),
|
|
877
|
-
).fetchone()
|
|
878
|
-
except sqlite3.Error:
|
|
879
|
-
return None
|
|
880
|
-
return str(row["value"]) if row and row["value"] else None
|
|
881
|
-
|
|
882
|
-
def mark_superseded(self, old_node_id: str, new_node_id: str) -> Dict[str, Any]:
|
|
883
|
-
"""Record that ``old_node_id`` was replaced by ``new_node_id``.
|
|
884
|
-
|
|
885
|
-
The old node stays queryable (knowledge is durable); readers can follow
|
|
886
|
-
the revision chain via ``nodes_v2.superseded_by``.
|
|
887
|
-
"""
|
|
888
|
-
with self._connect() as conn:
|
|
889
|
-
for node_id in (old_node_id, new_node_id):
|
|
890
|
-
exists = conn.execute(
|
|
891
|
-
"SELECT 1 FROM nodes_v2 WHERE id=?", (node_id,)
|
|
892
|
-
).fetchone()
|
|
893
|
-
if not exists:
|
|
894
|
-
raise FileNotFoundError(node_id)
|
|
895
|
-
conn.execute(
|
|
896
|
-
"UPDATE nodes_v2 SET superseded_by=?, updated_at=? WHERE id=?",
|
|
897
|
-
(new_node_id, _now(), old_node_id),
|
|
898
|
-
)
|
|
899
|
-
return {"status": "ok", "node_id": old_node_id, "superseded_by": new_node_id}
|
|
900
|
-
|
|
901
497
|
def _v2_delete_nodes(self, conn: sqlite3.Connection, ids) -> None:
|
|
902
498
|
"""Mirror legacy node deletions into v2 (edges_v2 cascade on the FK)."""
|
|
903
499
|
if KGStoreV2 is None:
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Retrieval: the read path from a query to grounded context.
|
|
2
|
+
|
|
3
|
+
v11.3.0 turned this module into a package. ``KnowledgeGraphRetrievalMixin`` is
|
|
4
|
+
now composed from four cohesive sub-mixins — graph view + lexical search,
|
|
5
|
+
the hybrid pipeline, context assembly, and destructive maintenance — each of
|
|
6
|
+
which moved here verbatim. Every name this module exported before still
|
|
7
|
+
resolves from ``lattice_brain.graph.retrieval``.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
# ruff: noqa: F403,F405
|
|
13
|
+
from .._kg_common import * # noqa: F403,F401
|
|
14
|
+
from ..fusion import ( # noqa: F401
|
|
15
|
+
DEFAULT_EXPANSION_CAP,
|
|
16
|
+
DEFAULT_EXPANSION_SEEDS,
|
|
17
|
+
expand_with_neighbors,
|
|
18
|
+
graph_expansion_enabled,
|
|
19
|
+
rrf_fuse,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
# --- Compat seam (v9.9.5 decomposition) -------------------------------------
|
|
23
|
+
# The non-search read surface (list_documents / workspaces_of /
|
|
24
|
+
# filter_scoped_nodes / neighbors / get_node / relationship_search /
|
|
25
|
+
# traverse / stats) moved byte-identically to .retrieval_reads as
|
|
26
|
+
# KnowledgeGraphReadsMixin. Re-exported here so any legacy
|
|
27
|
+
# ``from lattice_brain.graph.retrieval import ...`` site keeps resolving.
|
|
28
|
+
from ..retrieval_reads import KnowledgeGraphReadsMixin # noqa: F401
|
|
29
|
+
from .context import _ContextMixin
|
|
30
|
+
from .graph_view import _GraphViewMixin
|
|
31
|
+
from .hybrid import _HybridSearchMixin
|
|
32
|
+
from .maintenance import _MaintenanceMixin
|
|
33
|
+
from .signals import ( # noqa: F401
|
|
34
|
+
MULTIMODAL_NODE_TYPES,
|
|
35
|
+
context_quality_signal,
|
|
36
|
+
multimodal_signal,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class KnowledgeGraphRetrievalMixin(
|
|
41
|
+
_ContextMixin,
|
|
42
|
+
_HybridSearchMixin,
|
|
43
|
+
_GraphViewMixin,
|
|
44
|
+
_MaintenanceMixin,
|
|
45
|
+
):
|
|
46
|
+
"""The graph read surface, composed from its four cohesive halves.
|
|
47
|
+
|
|
48
|
+
The sub-mixins define disjoint method sets, so resolution order changes
|
|
49
|
+
nothing at runtime: this class exposes exactly the methods it exposed when
|
|
50
|
+
they all lived in one 1,120-line module. The order is written
|
|
51
|
+
most-composed-first (context → hybrid → graph view) because each half
|
|
52
|
+
names the one below it as its typing-only base, and C3 needs a subclass
|
|
53
|
+
ahead of the class it extends.
|
|
54
|
+
"""
|