ltcai 11.2.0 → 11.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -53
- package/docs/CHANGELOG.md +61 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/MULTI_AGENT_RUNTIME.md +1 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +6 -2
- package/docs/PERMISSION_MODE.md +1 -1
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/kg-schema.md +2 -2
- package/docs/v11.3.0_PLAN.md +202 -0
- package/docs/v11.4.0_RUST_FOUNDATION_PLAN.md +176 -0
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/graph/_kg_common/__init__.py +287 -0
- package/lattice_brain/graph/_kg_common/extraction.py +516 -0
- package/lattice_brain/graph/_kg_common/relations.py +161 -0
- package/lattice_brain/graph/_kg_common/text.py +479 -0
- package/lattice_brain/graph/discovery_index/__init__.py +35 -0
- package/lattice_brain/graph/discovery_index/cleanup.py +182 -0
- package/lattice_brain/graph/discovery_index/extract.py +137 -0
- package/lattice_brain/graph/discovery_index/scan.py +411 -0
- package/lattice_brain/graph/discovery_index/upsert.py +495 -0
- package/lattice_brain/graph/projection/__init__.py +42 -0
- package/lattice_brain/graph/projection/curation.py +500 -0
- package/lattice_brain/graph/{projection.py → projection/v2_schema.py} +15 -477
- package/lattice_brain/graph/retrieval/__init__.py +54 -0
- package/lattice_brain/graph/retrieval/context.py +197 -0
- package/lattice_brain/graph/retrieval/graph_view.py +319 -0
- package/lattice_brain/graph/retrieval/hybrid.py +488 -0
- package/lattice_brain/graph/retrieval/maintenance.py +121 -0
- package/lattice_brain/graph/retrieval/signals.py +95 -0
- package/lattice_brain/graph/retrieval_vector/__init__.py +42 -0
- package/lattice_brain/graph/retrieval_vector/fingerprint.py +97 -0
- package/lattice_brain/graph/retrieval_vector/indexing.py +347 -0
- package/lattice_brain/graph/retrieval_vector/search.py +560 -0
- package/lattice_brain/graph/retrieval_vector/status.py +374 -0
- package/lattice_brain/ingestion/__init__.py +130 -0
- package/lattice_brain/ingestion/_contract.py +90 -0
- package/lattice_brain/ingestion/constants.py +127 -0
- package/lattice_brain/ingestion/folder_scan.py +57 -0
- package/lattice_brain/ingestion/folders.py +258 -0
- package/lattice_brain/ingestion/hashing.py +26 -0
- package/lattice_brain/ingestion/jobs_api.py +107 -0
- package/lattice_brain/ingestion/models.py +80 -0
- package/lattice_brain/ingestion/pipeline.py +486 -0
- package/lattice_brain/ingestion/quality.py +209 -0
- package/lattice_brain/ingestion/routing.py +295 -0
- package/lattice_brain/multimodal/__init__.py +164 -0
- package/lattice_brain/multimodal/audio.py +77 -0
- package/lattice_brain/multimodal/common.py +118 -0
- package/lattice_brain/multimodal/images.py +498 -0
- package/lattice_brain/multimodal/ports.py +169 -0
- package/lattice_brain/multimodal/video.py +410 -0
- package/lattice_brain/portability/__init__.py +90 -0
- package/lattice_brain/portability/_contract.py +42 -0
- package/lattice_brain/portability/backups.py +338 -0
- package/lattice_brain/portability/bundles.py +136 -0
- package/lattice_brain/portability/constants.py +93 -0
- package/lattice_brain/portability/fsops.py +138 -0
- package/lattice_brain/portability/service.py +41 -0
- package/lattice_brain/{portability.py → portability/sharing.py} +44 -677
- package/lattice_brain/runtime/__init__.py +1 -1
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/chronicle.py +63 -0
- package/latticeai/core/agent/__init__.py +93 -0
- package/latticeai/core/agent/_contract.py +79 -0
- package/latticeai/core/agent/context.py +57 -0
- package/latticeai/core/agent/deps.py +125 -0
- package/latticeai/core/agent/execution.py +622 -0
- package/latticeai/core/agent/planning.py +145 -0
- package/latticeai/core/agent/recovery.py +157 -0
- package/latticeai/core/agent/runtime.py +210 -0
- package/latticeai/core/agent/verification.py +231 -0
- package/latticeai/core/embedding_providers/__init__.py +151 -0
- package/latticeai/core/embedding_providers/base.py +199 -0
- package/latticeai/core/embedding_providers/captions.py +162 -0
- package/latticeai/core/embedding_providers/profiles.py +126 -0
- package/latticeai/core/embedding_providers/text.py +350 -0
- package/latticeai/core/embedding_providers/vision.py +352 -0
- package/latticeai/core/file_generation/__init__.py +115 -0
- package/latticeai/core/file_generation/bundles.py +76 -0
- package/latticeai/core/file_generation/extraction.py +154 -0
- package/latticeai/core/file_generation/inference.py +235 -0
- package/latticeai/core/file_generation/orchestration.py +152 -0
- package/latticeai/core/file_generation/prompting.py +117 -0
- package/latticeai/core/file_generation/repair.py +114 -0
- package/latticeai/core/file_generation/sanitize.py +61 -0
- package/latticeai/core/file_generation/validation.py +201 -0
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/messages.py +9 -0
- package/latticeai/core/workspace_os_constants.py +1 -1
- package/latticeai/integrations/telegram_bot/__init__.py +123 -0
- package/latticeai/integrations/telegram_bot/__main__.py +17 -0
- package/latticeai/integrations/telegram_bot/config.py +86 -0
- package/latticeai/integrations/telegram_bot/dispatch.py +311 -0
- package/latticeai/integrations/telegram_bot/flows.py +478 -0
- package/latticeai/integrations/telegram_bot/helpers.py +322 -0
- package/latticeai/integrations/telegram_bot/screens.py +394 -0
- package/latticeai/models/router/__init__.py +88 -0
- package/latticeai/models/router/_contract.py +66 -0
- package/latticeai/models/router/branding.py +56 -0
- package/latticeai/models/router/catalog.py +69 -0
- package/latticeai/models/router/documents.py +199 -0
- package/latticeai/models/router/errors.py +37 -0
- package/latticeai/models/router/generation.py +258 -0
- package/latticeai/models/router/loading.py +291 -0
- package/latticeai/models/router/local_models.py +85 -0
- package/latticeai/models/router/registry.py +147 -0
- package/latticeai/runtime/build_phases/__init__.py +82 -0
- package/latticeai/runtime/build_phases/features.py +407 -0
- package/latticeai/runtime/build_phases/foundation.py +555 -0
- package/latticeai/runtime/build_phases/web.py +492 -0
- package/latticeai/runtime/runtime_context.py +1 -0
- package/latticeai/services/architecture_readiness.py +48 -19
- package/latticeai/services/brain_intelligence/__init__.py +58 -0
- package/latticeai/services/brain_intelligence/_contract.py +71 -0
- package/latticeai/services/brain_intelligence/consistency.py +193 -0
- package/latticeai/services/brain_intelligence/constants.py +47 -0
- package/latticeai/services/brain_intelligence/digest.py +258 -0
- package/latticeai/services/brain_intelligence/health.py +331 -0
- package/latticeai/services/brain_intelligence/proposals.py +264 -0
- package/latticeai/services/brain_intelligence/sampling.py +84 -0
- package/latticeai/services/brain_intelligence/service.py +48 -0
- package/latticeai/services/chronicle.py +557 -0
- package/latticeai/services/memory_service/__init__.py +52 -0
- package/latticeai/services/memory_service/_contract.py +100 -0
- package/latticeai/services/memory_service/brief.py +431 -0
- package/latticeai/services/memory_service/constants.py +57 -0
- package/latticeai/services/memory_service/maintenance.py +138 -0
- package/latticeai/services/memory_service/manager.py +186 -0
- package/latticeai/services/memory_service/proof.py +136 -0
- package/latticeai/services/memory_service/recall.py +225 -0
- package/latticeai/services/memory_service/service.py +48 -0
- package/latticeai/services/memory_service/stores.py +110 -0
- package/latticeai/services/model_runtime/__init__.py +322 -0
- package/latticeai/services/model_runtime/cloud.py +87 -0
- package/latticeai/services/model_runtime/download.py +282 -0
- package/latticeai/services/model_runtime/engines.py +341 -0
- package/latticeai/services/model_runtime/loading.py +178 -0
- package/latticeai/services/model_runtime/service.py +129 -0
- package/latticeai/services/model_runtime/state.py +131 -0
- package/latticeai/services/model_runtime/status.py +255 -0
- package/latticeai/services/product_readiness.py +15 -7
- package/latticeai/setup/wizard/__init__.py +126 -0
- package/latticeai/setup/wizard/catalog.py +172 -0
- package/latticeai/setup/wizard/detect.py +323 -0
- package/latticeai/setup/wizard/install.py +348 -0
- package/latticeai/setup/wizard/paths.py +168 -0
- package/latticeai/setup/wizard/plans.py +74 -0
- package/latticeai/setup/wizard/recommend.py +320 -0
- package/package.json +6 -2
- package/scripts/bump_version.py +14 -0
- package/scripts/capture_release_evidence.mjs +33 -21
- package/scripts/check_current_release_docs.mjs +1 -1
- package/scripts/check_i18n_namespace_coverage.mjs +41 -4
- package/scripts/check_max_file_lines.mjs +102 -0
- package/scripts/check_release_evidence_bound.mjs +30 -15
- package/scripts/check_screenshot_pixel_delta.py +34 -4
- package/scripts/check_server_i18n.mjs +1 -0
- package/scripts/generate_rust_parity_fixtures.py +562 -0
- package/scripts/lib/mock_server_fingerprint.mjs +94 -0
- package/scripts/release_screen_claims.json +31 -2
- package/src-tauri/Cargo.lock +361 -3
- package/src-tauri/Cargo.toml +6 -1
- package/src-tauri/src/backend.rs +349 -0
- package/src-tauri/src/folder.rs +33 -0
- package/src-tauri/src/main.rs +97 -399
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +41 -37
- package/static/app/assets/Act-yYpYnn0v.js +1 -0
- package/static/app/assets/AdminConsole-DL3Cr5pL.js +1 -0
- package/static/app/assets/{Brain-tuhI4sOC.js → Brain-C1HBN0Wf.js} +2 -2
- package/static/app/assets/BrainHome-DoXRhUUC.js +2 -0
- package/static/app/assets/BrainSignals-6yR6ir5t.js +1 -0
- package/static/app/assets/Capture-CFIRsFNE.js +1 -0
- package/static/app/assets/Chronicle-BZbEgiwN.js +1 -0
- package/static/app/assets/CommandPalette-D2pMxC2I.js +1 -0
- package/static/app/assets/Library-DwO3yZST.js +1 -0
- package/static/app/assets/{LivingBrain-DBwhto14.js → LivingBrain-Jn1GK0-S.js} +1 -1
- package/static/app/assets/ProductFlow-B-w1R4Oo.js +1 -0
- package/static/app/assets/ReviewCard-6B27X8Vg.js +3 -0
- package/static/app/assets/System-DW8F-2xL.js +1 -0
- package/static/app/assets/arrow-left-DXvKg9U6.js +1 -0
- package/static/app/assets/{bot-Cia42c2h.js → bot-IM_E_Y12.js} +1 -1
- package/static/app/assets/brain-Ci1CkWjM.js +1 -0
- package/static/app/assets/{button-2j2Ijzgq.js → button-COwyqfHM.js} +1 -1
- package/static/app/assets/circle-check-DfInj-qD.js +1 -0
- package/static/app/assets/{circle-pause-BEFeWpVW.js → circle-pause-DEM4A1Y5.js} +1 -1
- package/static/app/assets/{circle-play-ujXMcHxl.js → circle-play-C9djDuLd.js} +1 -1
- package/static/app/assets/{cpu-k4awryFq.js → cpu-DFdo1gw-.js} +1 -1
- package/static/app/assets/{download-DFbLJ_ig.js → download-SnJL6oqk.js} +1 -1
- package/static/app/assets/{folder-open-7y_b6xkM.js → folder-open-CqZeDkjE.js} +1 -1
- package/static/app/assets/{hard-drive-Bidh02Kr.js → hard-drive-j1jJXYYf.js} +1 -1
- package/static/app/assets/{index-DwDl9-8Y.css → index-BLPb5lmE.css} +1 -1
- package/static/app/assets/index-_u5iUHDr.js +10 -0
- package/static/app/assets/input-B0lPdRQZ.js +1 -0
- package/static/app/assets/link-2-CoFbooHS.js +1 -0
- package/static/app/assets/{permissionCopy-Bpb83Hx9.js → permissionCopy-BsyLxtao.js} +1 -1
- package/static/app/assets/primitives-DEbN-d6p.js +1 -0
- package/static/app/assets/search-BybIWPNd.js +1 -0
- package/static/app/assets/{share-2-BH1M-WNi.js → share-2-CVtZ_ewX.js} +1 -1
- package/static/app/assets/{shield-alert-BlKdBXcG.js → shield-alert-CBi2GNWM.js} +1 -1
- package/static/app/assets/{textarea-CCWbUfFB.js → textarea-DNMpB5ih.js} +1 -1
- package/static/app/assets/{useFocusTrap-YdHQ7pJ1.js → useFocusTrap-C83t3GXF.js} +1 -1
- package/static/app/assets/useMutation-DtbJDoyz.js +1 -0
- package/static/app/assets/{useQuery-CXQiwbVT.js → useQuery-Dcp1OChy.js} +1 -1
- package/static/app/assets/utils-BlZr7Pd4.js +4 -0
- package/static/app/assets/workspace-jJY4RuAV.js +1 -0
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/lattice_brain/graph/_kg_common.py +0 -1331
- package/lattice_brain/graph/discovery_index.py +0 -1141
- package/lattice_brain/graph/retrieval.py +0 -1120
- package/lattice_brain/graph/retrieval_vector.py +0 -1293
- package/lattice_brain/ingestion.py +0 -1525
- package/lattice_brain/multimodal.py +0 -1258
- package/latticeai/core/agent.py +0 -1465
- package/latticeai/core/embedding_providers.py +0 -1196
- package/latticeai/core/file_generation.py +0 -1047
- package/latticeai/integrations/telegram_bot.py +0 -1390
- package/latticeai/models/router.py +0 -1007
- package/latticeai/runtime/build_phases.py +0 -1450
- package/latticeai/services/brain_intelligence.py +0 -1083
- package/latticeai/services/memory_service.py +0 -1177
- package/latticeai/services/model_runtime.py +0 -1281
- package/latticeai/setup/wizard.py +0 -1310
- package/static/app/assets/Act-AWf0SAKp.js +0 -1
- package/static/app/assets/AdminConsole-D0u8Tiyj.js +0 -1
- package/static/app/assets/BrainHome-Ts7G_Ila.js +0 -2
- package/static/app/assets/BrainSignals-jMYgQ2Ar.js +0 -1
- package/static/app/assets/Capture-CqOSzyPr.js +0 -1
- package/static/app/assets/CommandPalette-DC0Bzh-I.js +0 -1
- package/static/app/assets/Library-CX-bbhmK.js +0 -1
- package/static/app/assets/ProductFlow-BHA2cfKI.js +0 -1
- package/static/app/assets/ReviewCard-BUhCKRNM.js +0 -3
- package/static/app/assets/System-Bu2t5hn1.js +0 -1
- package/static/app/assets/arrow-left-Dzwa5zRb.js +0 -1
- package/static/app/assets/brain-DJMoqrwx.js +0 -1
- package/static/app/assets/index-BpYkzcVm.js +0 -10
- package/static/app/assets/input-DSlJJxRs.js +0 -1
- package/static/app/assets/primitives-BCx6TvfG.js +0 -1
- package/static/app/assets/search-Cgy8cCFJ.js +0 -1
- package/static/app/assets/utils-zqPZJxdx.js +0 -4
- package/static/app/assets/workspace-DXTihhfU.js +0 -1
|
@@ -1,398 +1,69 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Logical export/import, and the opt-in Brain Network subgraph share.
|
|
2
2
|
|
|
3
|
-
The
|
|
4
|
-
|
|
3
|
+
The half of :class:`~lattice_brain.portability.KGPortabilityService` that moves
|
|
4
|
+
*knowledge* rather than *files*: a whole-graph JSON artifact, and — behind the
|
|
5
|
+
``LATTICEAI_BRAIN_NETWORK`` opt-in — a signed, encrypted slice of the graph that
|
|
6
|
+
arrives at the far end as review proposals, never as a merge.
|
|
5
7
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
artifact; the importer re-embeds with its own embedder and reports the
|
|
9
|
-
resulting index state under ``result["index"]`` (``degraded: true`` means the
|
|
10
|
-
content landed but recall is lexical-only until a rebuild succeeds). That is
|
|
11
|
-
what makes it portable across machines.
|
|
12
|
-
* **Binary backup/restore** (ZIP): a faithful snapshot of the SQLite DB (incl.
|
|
13
|
-
vector embeddings) plus the blob directory, integrity-checked, for
|
|
14
|
-
same-machine recovery.
|
|
8
|
+
The receiving X25519 key is loaded here (``load_recipient_identity``), so a test
|
|
9
|
+
standing in for it patches *this* module rather than the package.
|
|
15
10
|
"""
|
|
16
11
|
|
|
17
12
|
from __future__ import annotations
|
|
18
13
|
|
|
19
14
|
import base64
|
|
20
|
-
import hashlib
|
|
21
15
|
import json
|
|
22
|
-
import logging
|
|
23
16
|
import os
|
|
24
|
-
import
|
|
25
|
-
import
|
|
26
|
-
import tempfile
|
|
27
|
-
import zipfile
|
|
28
|
-
from contextlib import closing
|
|
29
|
-
from pathlib import Path, PurePosixPath
|
|
30
|
-
from typing import Any, Dict, Iterable, List, Optional, Set
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
31
19
|
|
|
32
20
|
from cryptography.exceptions import InvalidTag
|
|
33
21
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
34
22
|
|
|
35
|
-
from
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
EncryptedBrainArchive,
|
|
39
|
-
_derive_key,
|
|
40
|
-
)
|
|
41
|
-
from .gates import FeatureGate
|
|
42
|
-
from .quiet import quiet
|
|
43
|
-
from .sealed_box import (
|
|
23
|
+
from ..archive import KDF_ITERATIONS, _derive_key
|
|
24
|
+
from ..quiet import quiet
|
|
25
|
+
from ..sealed_box import (
|
|
44
26
|
SEALED_BOX_ALGORITHM,
|
|
45
27
|
load_recipient_identity,
|
|
46
28
|
public_key_fingerprint,
|
|
47
29
|
seal,
|
|
48
30
|
)
|
|
49
|
-
from
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
BACKUP_FORMAT = "latticeai.kg.backup"
|
|
60
|
-
|
|
61
|
-
# ── selective subgraph share (v11.1.0 prototype, off by default) ─────────────
|
|
62
|
-
#: Logical format of a *partial* bundle: a chosen slice of the graph plus its
|
|
63
|
-
#: provenance, signed by this device. Deliberately distinct from ``FORMAT`` so
|
|
64
|
-
#: the whole-graph importer can never be handed a subgraph by accident.
|
|
65
|
-
SUBGRAPH_FORMAT = "latticeai.kg.subgraph"
|
|
66
|
-
SUBGRAPH_FORMAT_VERSION = 1
|
|
67
|
-
#: Encrypted on-disk envelope for the same bundle, reusing the ``.latticebrain``
|
|
68
|
-
#: passphrase mechanism (PBKDF2-SHA256 + AES-256-GCM) from ``archive.py``.
|
|
69
|
-
SUBGRAPH_ARCHIVE_FORMAT = "latticebrain.subgraph"
|
|
70
|
-
#: Opt-in gate. Off unless the operator sets it: sharing knowledge with another
|
|
71
|
-
#: Brain is a network act, and local-first means those are never on by default.
|
|
72
|
-
BRAIN_NETWORK_ENV = "LATTICEAI_BRAIN_NETWORK"
|
|
73
|
-
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
|
74
|
-
#: How a bundle can be encrypted. ``passphrase`` needs a secret agreed out of
|
|
75
|
-
#: band; ``recipient_public_key`` needs nothing secret to travel at all.
|
|
76
|
-
ENCRYPTION_MODES = ("passphrase", "recipient_public_key")
|
|
77
|
-
#: A received bundle becomes review items, never a merge. The cap keeps one
|
|
78
|
-
#: peer from flooding the inbox; what it dropped is reported, not hidden.
|
|
79
|
-
SUBGRAPH_PROPOSAL_CAP = 200
|
|
80
|
-
#: Review-queue vocabulary. ``kg_change_digest`` is the existing source for
|
|
81
|
-
#: "the graph would change"; the kind distinguishes shared-subgraph items.
|
|
82
|
-
SUBGRAPH_REVIEW_SOURCE = "kg_change_digest"
|
|
83
|
-
SUBGRAPH_REVIEW_KIND = "shared_subgraph_node"
|
|
84
|
-
#: Mirrors ``services.review_queue.OPEN_STATUSES``. Duplicated as a literal
|
|
85
|
-
#: rather than imported because ``lattice_brain`` must not depend on the
|
|
86
|
-
#: ``latticeai`` service layer; the review sink is injected, not imported.
|
|
87
|
-
_OPEN_REVIEW_STATUSES = frozenset({"pending", "snoozed"})
|
|
88
|
-
|
|
89
|
-
BRAIN_NETWORK_DISABLED_DETAIL = (
|
|
90
|
-
"Brain Network sharing is off. It is opt-in by design: set "
|
|
91
|
-
f"{BRAIN_NETWORK_ENV}=1 to enable selective subgraph export and receipt."
|
|
31
|
+
from ..utils import utc_now_iso
|
|
32
|
+
from ._contract import PortabilityCore as _Core
|
|
33
|
+
from .bundles import (
|
|
34
|
+
_canonical_digest,
|
|
35
|
+
_expand_neighbors,
|
|
36
|
+
_redact_node,
|
|
37
|
+
_redact_provenance_row,
|
|
38
|
+
_scope_node,
|
|
39
|
+
_select_node_ids,
|
|
40
|
+
_shared_node_summary,
|
|
92
41
|
)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
BRAIN_NETWORK_GATE = FeatureGate(
|
|
42
|
+
from .constants import (
|
|
43
|
+
_OPEN_REVIEW_STATUSES,
|
|
44
|
+
_PAYLOAD_KEYS,
|
|
45
|
+
_REDACTED_FIELDS,
|
|
46
|
+
BRAIN_NETWORK_DISABLED_DETAIL,
|
|
99
47
|
BRAIN_NETWORK_ENV,
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
48
|
+
BRAIN_NETWORK_GATE,
|
|
49
|
+
ENCRYPTION_MODES,
|
|
50
|
+
FORMAT,
|
|
51
|
+
FORMAT_VERSION,
|
|
52
|
+
SUBGRAPH_ARCHIVE_FORMAT,
|
|
53
|
+
SUBGRAPH_FORMAT,
|
|
54
|
+
SUBGRAPH_FORMAT_VERSION,
|
|
55
|
+
SUBGRAPH_PROPOSAL_CAP,
|
|
56
|
+
SUBGRAPH_REVIEW_KIND,
|
|
57
|
+
SUBGRAPH_REVIEW_SOURCE,
|
|
58
|
+
_stamp,
|
|
59
|
+
brain_network_enabled,
|
|
60
|
+
require_brain_network,
|
|
103
61
|
)
|
|
104
62
|
|
|
105
63
|
|
|
106
|
-
class
|
|
107
|
-
"""
|
|
108
|
-
|
|
109
|
-
def __init__(self, detail: str = BRAIN_NETWORK_DISABLED_DETAIL) -> None:
|
|
110
|
-
super().__init__(detail)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def brain_network_enabled() -> bool:
|
|
114
|
-
"""True only when the operator explicitly opted in (default: off)."""
|
|
115
|
-
return BRAIN_NETWORK_GATE.enabled()
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def require_brain_network() -> None:
|
|
119
|
-
if not brain_network_enabled():
|
|
120
|
-
raise BrainNetworkDisabled()
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def _stamp() -> str:
|
|
124
|
-
return utc_now_iso().replace(":", "").replace("-", "").replace(".", "")[:15]
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def _safe_zip_names(names) -> None:
|
|
128
|
-
for name in names:
|
|
129
|
-
path = PurePosixPath(name)
|
|
130
|
-
if path.is_absolute() or ".." in path.parts:
|
|
131
|
-
raise ValueError(f"Backup archive contains unsafe path: {name}")
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def _pre_restore_backup_dir(anchor: Path) -> Path:
|
|
135
|
-
backup_dir = anchor.parent / f"{anchor.name}.pre-restore-{_stamp()}"
|
|
136
|
-
index = 1
|
|
137
|
-
while backup_dir.exists():
|
|
138
|
-
backup_dir = anchor.parent / f"{anchor.name}.pre-restore-{_stamp()}-{index}"
|
|
139
|
-
index += 1
|
|
140
|
-
backup_dir.mkdir(parents=True)
|
|
141
|
-
return backup_dir
|
|
64
|
+
class KGPortabilitySharingMixin(_Core):
|
|
65
|
+
"""Export / import / share. Mixed into ``KGPortabilityService``."""
|
|
142
66
|
|
|
143
|
-
|
|
144
|
-
def _sqlite_siblings(db_path: Path) -> tuple[Path, Path, Path]:
|
|
145
|
-
return (db_path, Path(str(db_path) + "-wal"), Path(str(db_path) + "-shm"))
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
def _checkpoint_sqlite(db_path: Path) -> None:
|
|
149
|
-
if not db_path.exists():
|
|
150
|
-
return
|
|
151
|
-
try:
|
|
152
|
-
with closing(sqlite3.connect(str(db_path))) as conn, conn:
|
|
153
|
-
conn.execute("PRAGMA wal_checkpoint(FULL)")
|
|
154
|
-
except sqlite3.Error:
|
|
155
|
-
# Best-effort only. Existing sibling backup/restore still preserves
|
|
156
|
-
# the WAL files if a live connection prevents a checkpoint.
|
|
157
|
-
return
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def _restore_sibling(path: Path, backup: Path) -> None:
|
|
161
|
-
try:
|
|
162
|
-
shutil.copy2(backup, path)
|
|
163
|
-
except FileNotFoundError:
|
|
164
|
-
# The backup vanished (or never existed — transient -wal/-shm): the
|
|
165
|
-
# honest reconstruction is "no such sibling", never a crash that
|
|
166
|
-
# masks the error that triggered the rollback.
|
|
167
|
-
path.unlink(missing_ok=True)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def _replace_sqlite_atomically(src: Path, dest: Path, backup_dir: Path) -> None:
|
|
171
|
-
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
172
|
-
tmp = dest.parent / f".{dest.name}.restore-{_stamp()}-{os.getpid()}.tmp"
|
|
173
|
-
shutil.copyfile(src, tmp)
|
|
174
|
-
backups: dict[Path, Path] = {}
|
|
175
|
-
try:
|
|
176
|
-
_checkpoint_sqlite(dest)
|
|
177
|
-
# -wal/-shm are transient: another live connection can checkpoint and
|
|
178
|
-
# remove them between exists() and the copy/unlink. Treat a vanished
|
|
179
|
-
# sibling as "nothing to preserve" instead of crashing the restore.
|
|
180
|
-
for sibling in _sqlite_siblings(dest):
|
|
181
|
-
backup = backup_dir / sibling.name
|
|
182
|
-
try:
|
|
183
|
-
shutil.copy2(sibling, backup)
|
|
184
|
-
except FileNotFoundError:
|
|
185
|
-
quiet()
|
|
186
|
-
continue
|
|
187
|
-
backups[sibling] = backup
|
|
188
|
-
for sibling in _sqlite_siblings(dest)[1:]:
|
|
189
|
-
sibling.unlink(missing_ok=True)
|
|
190
|
-
os.replace(tmp, dest)
|
|
191
|
-
except Exception:
|
|
192
|
-
# Recovery I/O must never replace the swap error being reported —
|
|
193
|
-
# an exception raised here would mask it (the CI-observed
|
|
194
|
-
# "[Errno 2]" over the real failure). Best-effort restore, then
|
|
195
|
-
# always re-raise the original.
|
|
196
|
-
try:
|
|
197
|
-
tmp.unlink(missing_ok=True)
|
|
198
|
-
except OSError as cleanup_exc:
|
|
199
|
-
logging.warning("restore tmp cleanup failed: %s", cleanup_exc)
|
|
200
|
-
for sibling in _sqlite_siblings(dest):
|
|
201
|
-
try:
|
|
202
|
-
_restore_sibling(sibling, backups.get(sibling, backup_dir / sibling.name))
|
|
203
|
-
except OSError as rollback_exc:
|
|
204
|
-
logging.warning(
|
|
205
|
-
"restore sibling rollback incomplete for %s: %s",
|
|
206
|
-
sibling, rollback_exc,
|
|
207
|
-
)
|
|
208
|
-
raise
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
def _rollback_sqlite_from_backup(dest: Path, backup_dir: Path) -> None:
|
|
212
|
-
for sibling in _sqlite_siblings(dest):
|
|
213
|
-
_restore_sibling(sibling, backup_dir / sibling.name)
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
def _replace_tree_with_backup(src: Optional[Path], dest: Path, backup_dir: Path) -> None:
|
|
217
|
-
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
218
|
-
staged = dest.parent / f".{dest.name}.restore-{_stamp()}-{os.getpid()}"
|
|
219
|
-
backup = backup_dir / dest.name
|
|
220
|
-
if src and src.exists():
|
|
221
|
-
shutil.copytree(src, staged)
|
|
222
|
-
else:
|
|
223
|
-
staged.mkdir(parents=True)
|
|
224
|
-
try:
|
|
225
|
-
if dest.exists():
|
|
226
|
-
shutil.copytree(dest, backup)
|
|
227
|
-
shutil.rmtree(dest)
|
|
228
|
-
os.replace(staged, dest)
|
|
229
|
-
except Exception:
|
|
230
|
-
# Same masking guard as the sqlite swap: rollback I/O is
|
|
231
|
-
# best-effort and the original failure always propagates.
|
|
232
|
-
try:
|
|
233
|
-
if staged.exists():
|
|
234
|
-
shutil.rmtree(staged)
|
|
235
|
-
if dest.exists():
|
|
236
|
-
shutil.rmtree(dest)
|
|
237
|
-
if backup.exists():
|
|
238
|
-
shutil.copytree(backup, dest)
|
|
239
|
-
except OSError as rollback_exc:
|
|
240
|
-
logging.warning("blob tree rollback incomplete: %s", rollback_exc)
|
|
241
|
-
raise
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
#: Sender-identifying fields stripped from a shared bundle by default. The
|
|
245
|
-
#: knowledge travels; "which account, on which machine, at which path" does not.
|
|
246
|
-
_REDACTED_FIELDS = frozenset({"owner", "user_email", "source_uri", "permissions"})
|
|
247
|
-
#: The signed payload's key set. One tuple so export and verify can never
|
|
248
|
-
#: disagree about what the digest covers.
|
|
249
|
-
_PAYLOAD_KEYS = (
|
|
250
|
-
"nodes", "chunk_nodes", "edges", "chunks", "knowledge_sources", "provenance",
|
|
251
|
-
)
|
|
252
|
-
#: Node types a one-hop expansion never admits — they describe the sender
|
|
253
|
-
#: (identity, local paths), not the knowledge being shared.
|
|
254
|
-
_NEIGHBOR_EXCLUDED_TYPES = frozenset({"Person", "Source"})
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
def _canonical_digest(payload: Dict[str, Any]) -> str:
|
|
258
|
-
"""sha256 over the canonical JSON of a bundle payload.
|
|
259
|
-
|
|
260
|
-
Pinned in the signed header so the Ed25519 signature covers the contents,
|
|
261
|
-
not just the manifest describing them.
|
|
262
|
-
"""
|
|
263
|
-
canonical = json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
|
264
|
-
return hashlib.sha256(canonical).hexdigest()
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def _node_source_type(node: Dict[str, Any]) -> str:
|
|
268
|
-
try:
|
|
269
|
-
metadata = json.loads(node.get("metadata_json") or "{}")
|
|
270
|
-
except (TypeError, ValueError):
|
|
271
|
-
return ""
|
|
272
|
-
if not isinstance(metadata, dict):
|
|
273
|
-
return ""
|
|
274
|
-
return str(metadata.get("source_type") or "").strip().lower()
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
def _select_node_ids(nodes: List[Dict[str, Any]], selection: Dict[str, Any]) -> Set[str]:
|
|
278
|
-
wanted_ids = set(selection["node_ids"])
|
|
279
|
-
wanted_types = set(selection["node_types"])
|
|
280
|
-
wanted_sources = set(selection["source_types"])
|
|
281
|
-
keep: Set[str] = set()
|
|
282
|
-
for node in nodes:
|
|
283
|
-
node_id = str(node.get("id"))
|
|
284
|
-
if node_id in wanted_ids:
|
|
285
|
-
keep.add(node_id)
|
|
286
|
-
continue
|
|
287
|
-
if wanted_types and str(node.get("type") or "").strip().lower() in wanted_types:
|
|
288
|
-
keep.add(node_id)
|
|
289
|
-
continue
|
|
290
|
-
if wanted_sources and _node_source_type(node) in wanted_sources:
|
|
291
|
-
keep.add(node_id)
|
|
292
|
-
return keep
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
def _expand_neighbors(
|
|
296
|
-
keep: Set[str], edges: List[Dict[str, Any]], known: Dict[str, str]
|
|
297
|
-
) -> Set[str]:
|
|
298
|
-
"""One hop out from the selection, in both directions, within the scope.
|
|
299
|
-
|
|
300
|
-
``Person`` and ``Source`` neighbours are never pulled in implicitly: a
|
|
301
|
-
Person node's label *is* an email address and a Source node's summary *is*
|
|
302
|
-
a local file path, so "share this decision and what it touches" would
|
|
303
|
-
quietly ship the sender's identity and directory layout. Naming one of
|
|
304
|
-
those node ids explicitly still exports it — that is a decision, not a
|
|
305
|
-
side effect.
|
|
306
|
-
"""
|
|
307
|
-
expanded = set(keep)
|
|
308
|
-
|
|
309
|
-
def _admit(candidate: str) -> None:
|
|
310
|
-
if candidate in known and known[candidate] not in _NEIGHBOR_EXCLUDED_TYPES:
|
|
311
|
-
expanded.add(candidate)
|
|
312
|
-
|
|
313
|
-
for edge in edges:
|
|
314
|
-
source = str(edge.get("from_node"))
|
|
315
|
-
target = str(edge.get("to_node"))
|
|
316
|
-
if source in keep:
|
|
317
|
-
_admit(target)
|
|
318
|
-
if target in keep:
|
|
319
|
-
_admit(source)
|
|
320
|
-
return expanded
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
def _strip_fields(raw_json: Optional[str]) -> Optional[str]:
|
|
324
|
-
if not raw_json:
|
|
325
|
-
return raw_json
|
|
326
|
-
try:
|
|
327
|
-
parsed = json.loads(raw_json)
|
|
328
|
-
except (TypeError, ValueError):
|
|
329
|
-
return raw_json
|
|
330
|
-
if not isinstance(parsed, dict):
|
|
331
|
-
return raw_json
|
|
332
|
-
cleaned = {k: v for k, v in parsed.items() if k not in _REDACTED_FIELDS}
|
|
333
|
-
return json.dumps(cleaned, ensure_ascii=False, sort_keys=True)
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
def _redact_node(node: Dict[str, Any]) -> Dict[str, Any]:
|
|
337
|
-
redacted = dict(node)
|
|
338
|
-
redacted["metadata_json"] = _strip_fields(node.get("metadata_json"))
|
|
339
|
-
redacted["raw_json"] = _strip_fields(node.get("raw_json"))
|
|
340
|
-
return redacted
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
def _redact_provenance_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
|
344
|
-
redacted = dict(row)
|
|
345
|
-
redacted["owner"] = None
|
|
346
|
-
redacted["source_uri"] = None
|
|
347
|
-
redacted["permissions_json"] = "{}"
|
|
348
|
-
redacted["metadata_json"] = _strip_fields(row.get("metadata_json"))
|
|
349
|
-
return redacted
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
def _scope_node(node: Dict[str, Any], workspace_id: str) -> Dict[str, Any]:
|
|
353
|
-
"""Stamp the accepting workspace onto an incoming node.
|
|
354
|
-
|
|
355
|
-
Received knowledge belongs to the workspace that accepted it. Without this
|
|
356
|
-
the projection would file it as legacy-global — machine-shared — which is a
|
|
357
|
-
wider scope than the reviewer chose.
|
|
358
|
-
"""
|
|
359
|
-
scoped = dict(node)
|
|
360
|
-
try:
|
|
361
|
-
metadata = json.loads(scoped.get("metadata_json") or "{}")
|
|
362
|
-
except (TypeError, ValueError):
|
|
363
|
-
metadata = {}
|
|
364
|
-
if not isinstance(metadata, dict):
|
|
365
|
-
metadata = {}
|
|
366
|
-
metadata["workspace_id"] = workspace_id
|
|
367
|
-
scoped["metadata_json"] = json.dumps(metadata, ensure_ascii=False, sort_keys=True)
|
|
368
|
-
return scoped
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
def _shared_node_summary(node: Dict[str, Any], verdict: Dict[str, Any]) -> str:
|
|
372
|
-
body = str(node.get("summary") or "").strip()
|
|
373
|
-
origin = verdict.get("origin") or "unknown device"
|
|
374
|
-
prefix = f"[{origin}] "
|
|
375
|
-
return (prefix + body)[:500] if body else prefix.strip()
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
class KGPortabilityService:
|
|
379
|
-
def __init__(self, *, knowledge_graph: Any, data_dir, enable_graph: bool = True, device_identity: Any = None) -> None:
|
|
380
|
-
self._kg = knowledge_graph
|
|
381
|
-
self._data_dir = Path(data_dir)
|
|
382
|
-
self._enable = bool(enable_graph)
|
|
383
|
-
self._exports_dir = self._data_dir / "workspace_exports"
|
|
384
|
-
# v4 sovereignty: when a DeviceIdentity is wired, exports are signed
|
|
385
|
-
# and imports record origin provenance. Pre-v4 unsigned bundles stay
|
|
386
|
-
# importable locally (origin='unsigned-legacy') — signatures are
|
|
387
|
-
# mandatory only on the Brain Network peer path.
|
|
388
|
-
self._identity = device_identity
|
|
389
|
-
# The receiving X25519 key is created lazily: a Brain that never asks
|
|
390
|
-
# anyone to seal anything to it should not be minting key material on
|
|
391
|
-
# every startup (v11.2.0).
|
|
392
|
-
self._recipient: Any = None
|
|
393
|
-
self._recipient_loaded = False
|
|
394
|
-
|
|
395
|
-
# ── recipient key (v11.2.0 sealed-box receipt) ───────────────────────────
|
|
396
67
|
def _recipient_identity(self) -> Any:
|
|
397
68
|
"""This Brain's X25519 receiving key, created on first use.
|
|
398
69
|
|
|
@@ -422,13 +93,6 @@ class KGPortabilityService:
|
|
|
422
93
|
}
|
|
423
94
|
return {"available": True, **identity.describe()}
|
|
424
95
|
|
|
425
|
-
def available(self) -> bool:
|
|
426
|
-
return self._enable and self._kg is not None
|
|
427
|
-
|
|
428
|
-
def _require(self) -> None:
|
|
429
|
-
if not self.available():
|
|
430
|
-
raise RuntimeError("Knowledge Graph is disabled (LATTICEAI_ENABLE_GRAPH).")
|
|
431
|
-
|
|
432
96
|
# ── logical export / import ──────────────────────────────────────────────
|
|
433
97
|
def export(
|
|
434
98
|
self,
|
|
@@ -480,7 +144,7 @@ class KGPortabilityService:
|
|
|
480
144
|
origin = "unsigned-legacy"
|
|
481
145
|
signature = artifact.get("signature")
|
|
482
146
|
if signature:
|
|
483
|
-
from
|
|
147
|
+
from ..graph.identity import verify_manifest
|
|
484
148
|
|
|
485
149
|
if not verify_manifest(artifact.get("header") or {}, signature):
|
|
486
150
|
raise ValueError("Bundle signature verification failed — refusing to import.")
|
|
@@ -844,7 +508,7 @@ class KGPortabilityService:
|
|
|
844
508
|
if not signature.get("signature"):
|
|
845
509
|
errors.append("bundle is unsigned")
|
|
846
510
|
else:
|
|
847
|
-
from
|
|
511
|
+
from ..graph.identity import verify_manifest
|
|
848
512
|
|
|
849
513
|
if not verify_manifest(header, signature):
|
|
850
514
|
errors.append("signature does not match the bundle header")
|
|
@@ -1048,300 +712,3 @@ class KGPortabilityService:
|
|
|
1048
712
|
except Exception: # noqa: BLE001 — "not found" is the answer, not a failure
|
|
1049
713
|
return False
|
|
1050
714
|
return True
|
|
1051
|
-
|
|
1052
|
-
# ── binary backup / restore ──────────────────────────────────────────────
|
|
1053
|
-
def backup(self, dest_path=None) -> Dict[str, Any]:
|
|
1054
|
-
self._require()
|
|
1055
|
-
self._exports_dir.mkdir(parents=True, exist_ok=True)
|
|
1056
|
-
dest = Path(dest_path) if dest_path else self._exports_dir / f"kg-backup-{_stamp()}.zip"
|
|
1057
|
-
with tempfile.TemporaryDirectory() as tmp_s:
|
|
1058
|
-
tmp = Path(tmp_s)
|
|
1059
|
-
db_copy = tmp / "knowledge_graph.sqlite"
|
|
1060
|
-
self._kg.backup_database(db_copy)
|
|
1061
|
-
manifest = {
|
|
1062
|
-
"format": BACKUP_FORMAT,
|
|
1063
|
-
"format_version": FORMAT_VERSION,
|
|
1064
|
-
**self._kg.schema_versions(),
|
|
1065
|
-
"created_at": utc_now_iso(),
|
|
1066
|
-
"db_sha256": _sha256_file(db_copy),
|
|
1067
|
-
"has_blobs": Path(self._kg.blob_dir).exists(),
|
|
1068
|
-
}
|
|
1069
|
-
with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
1070
|
-
zf.write(db_copy, "knowledge_graph.sqlite")
|
|
1071
|
-
zf.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
|
|
1072
|
-
blob_dir = Path(self._kg.blob_dir)
|
|
1073
|
-
if blob_dir.exists():
|
|
1074
|
-
for f in blob_dir.rglob("*"):
|
|
1075
|
-
if f.is_file():
|
|
1076
|
-
zf.write(f, f"blobs/{f.relative_to(blob_dir)}")
|
|
1077
|
-
return {"path": str(dest), "bytes": dest.stat().st_size, "manifest": manifest}
|
|
1078
|
-
|
|
1079
|
-
def restore(
|
|
1080
|
-
self,
|
|
1081
|
-
archive_path,
|
|
1082
|
-
*,
|
|
1083
|
-
verify: bool = True,
|
|
1084
|
-
dry_run: bool = False,
|
|
1085
|
-
confirm: bool = False,
|
|
1086
|
-
) -> Dict[str, Any]:
|
|
1087
|
-
self._require()
|
|
1088
|
-
archive = Path(archive_path)
|
|
1089
|
-
if not archive.exists():
|
|
1090
|
-
raise FileNotFoundError(f"Backup archive not found: {archive}")
|
|
1091
|
-
if not dry_run and not confirm:
|
|
1092
|
-
raise ValueError("Explicit confirmation is required before restoring a Knowledge Graph backup.")
|
|
1093
|
-
with zipfile.ZipFile(archive) as zf:
|
|
1094
|
-
names = zf.namelist()
|
|
1095
|
-
_safe_zip_names(names)
|
|
1096
|
-
if "knowledge_graph.sqlite" not in names:
|
|
1097
|
-
raise ValueError("Archive is missing knowledge_graph.sqlite.")
|
|
1098
|
-
manifest = json.loads(zf.read("manifest.json")) if "manifest.json" in names else {}
|
|
1099
|
-
with tempfile.TemporaryDirectory() as tmp_s:
|
|
1100
|
-
tmp = Path(tmp_s)
|
|
1101
|
-
zf.extractall(tmp)
|
|
1102
|
-
db_src = tmp / "knowledge_graph.sqlite"
|
|
1103
|
-
if verify and manifest.get("db_sha256"):
|
|
1104
|
-
if _sha256_file(db_src) != manifest["db_sha256"]:
|
|
1105
|
-
raise ValueError("Backup integrity check failed (db sha256 mismatch).")
|
|
1106
|
-
if dry_run:
|
|
1107
|
-
return {
|
|
1108
|
-
"restored": False,
|
|
1109
|
-
"dry_run": True,
|
|
1110
|
-
"verified": True,
|
|
1111
|
-
"manifest": manifest,
|
|
1112
|
-
"planned": {
|
|
1113
|
-
"database": str(self._kg.db_path),
|
|
1114
|
-
"blobs": str(self._kg.blob_dir),
|
|
1115
|
-
"archive": str(archive),
|
|
1116
|
-
},
|
|
1117
|
-
}
|
|
1118
|
-
db_dest = Path(self._kg.db_path)
|
|
1119
|
-
blob_dest = Path(self._kg.blob_dir)
|
|
1120
|
-
backup_dir = _pre_restore_backup_dir(db_dest)
|
|
1121
|
-
try:
|
|
1122
|
-
_replace_sqlite_atomically(db_src, db_dest, backup_dir)
|
|
1123
|
-
blob_src = tmp / "blobs"
|
|
1124
|
-
_replace_tree_with_backup(blob_src if blob_src.exists() else None, blob_dest, backup_dir)
|
|
1125
|
-
except Exception:
|
|
1126
|
-
# The rollback is best-effort recovery; an I/O slip inside
|
|
1127
|
-
# it must never mask the restore failure being reported.
|
|
1128
|
-
try:
|
|
1129
|
-
_rollback_sqlite_from_backup(db_dest, backup_dir)
|
|
1130
|
-
except OSError as rollback_exc:
|
|
1131
|
-
logging.warning(
|
|
1132
|
-
"pre-restore rollback incomplete (backup kept at %s): %s",
|
|
1133
|
-
backup_dir, rollback_exc,
|
|
1134
|
-
)
|
|
1135
|
-
raise
|
|
1136
|
-
stats = self._kg.stats()
|
|
1137
|
-
return {
|
|
1138
|
-
"restored": True,
|
|
1139
|
-
"manifest": manifest,
|
|
1140
|
-
"pre_restore_backup": str(backup_dir),
|
|
1141
|
-
"nodes": sum(stats.get("nodes", {}).values()),
|
|
1142
|
-
}
|
|
1143
|
-
|
|
1144
|
-
def verify_backup(self, archive_path) -> Dict[str, Any]:
|
|
1145
|
-
archive = Path(archive_path)
|
|
1146
|
-
if not archive.exists():
|
|
1147
|
-
return {"ok": False, "path": str(archive), "errors": [f"Backup archive not found: {archive}"]}
|
|
1148
|
-
try:
|
|
1149
|
-
with zipfile.ZipFile(archive) as zf:
|
|
1150
|
-
names = zf.namelist()
|
|
1151
|
-
_safe_zip_names(names)
|
|
1152
|
-
if "knowledge_graph.sqlite" not in names:
|
|
1153
|
-
raise ValueError("Archive is missing knowledge_graph.sqlite.")
|
|
1154
|
-
manifest = json.loads(zf.read("manifest.json")) if "manifest.json" in names else {}
|
|
1155
|
-
with tempfile.TemporaryDirectory() as tmp_s:
|
|
1156
|
-
tmp = Path(tmp_s)
|
|
1157
|
-
zf.extract("knowledge_graph.sqlite", tmp)
|
|
1158
|
-
db_src = tmp / "knowledge_graph.sqlite"
|
|
1159
|
-
if manifest.get("db_sha256") and _sha256_file(db_src) != manifest["db_sha256"]:
|
|
1160
|
-
raise ValueError("Backup integrity check failed (db sha256 mismatch).")
|
|
1161
|
-
return {"ok": True, "path": str(archive), "manifest": manifest, "errors": []}
|
|
1162
|
-
except (ValueError, zipfile.BadZipFile, OSError, json.JSONDecodeError) as exc:
|
|
1163
|
-
return {"ok": False, "path": str(archive), "errors": [str(exc)]}
|
|
1164
|
-
|
|
1165
|
-
# ── encrypted .latticebrain archive ───────────────────────────────────
|
|
1166
|
-
def encrypted_archive(self, dest_path=None, *, passphrase: str) -> Dict[str, Any]:
|
|
1167
|
-
self._require()
|
|
1168
|
-
self._exports_dir.mkdir(parents=True, exist_ok=True)
|
|
1169
|
-
dest = Path(dest_path) if dest_path else self._exports_dir / f"brain-{_stamp()}.latticebrain"
|
|
1170
|
-
metadata = {
|
|
1171
|
-
"storage": self.storage_status().get("active", {}),
|
|
1172
|
-
"snapshot": self.snapshot_metadata(),
|
|
1173
|
-
"device_identity": self._identity.describe() if self._identity is not None else {},
|
|
1174
|
-
"provenance": {"exported_at": utc_now_iso(), "source": "kg-portability"},
|
|
1175
|
-
}
|
|
1176
|
-
archive = EncryptedBrainArchive(
|
|
1177
|
-
BrainArchivePaths(
|
|
1178
|
-
db_path=Path(self._kg.db_path),
|
|
1179
|
-
blob_dir=Path(self._kg.blob_dir),
|
|
1180
|
-
data_dir=self._data_dir,
|
|
1181
|
-
metadata=metadata,
|
|
1182
|
-
)
|
|
1183
|
-
)
|
|
1184
|
-
return archive.create(dest, passphrase=passphrase)
|
|
1185
|
-
|
|
1186
|
-
def inspect_encrypted_archive(self, archive_path, *, passphrase: Optional[str] = None) -> Dict[str, Any]:
|
|
1187
|
-
archive = EncryptedBrainArchive(
|
|
1188
|
-
BrainArchivePaths(
|
|
1189
|
-
db_path=Path(self._kg.db_path),
|
|
1190
|
-
blob_dir=Path(self._kg.blob_dir),
|
|
1191
|
-
data_dir=self._data_dir,
|
|
1192
|
-
)
|
|
1193
|
-
)
|
|
1194
|
-
return archive.inspect(Path(archive_path), passphrase=passphrase)
|
|
1195
|
-
|
|
1196
|
-
def verify_encrypted_archive(self, archive_path, *, passphrase: str) -> Dict[str, Any]:
|
|
1197
|
-
archive = EncryptedBrainArchive(
|
|
1198
|
-
BrainArchivePaths(
|
|
1199
|
-
db_path=Path(self._kg.db_path),
|
|
1200
|
-
blob_dir=Path(self._kg.blob_dir),
|
|
1201
|
-
data_dir=self._data_dir,
|
|
1202
|
-
)
|
|
1203
|
-
)
|
|
1204
|
-
return archive.verify(Path(archive_path), passphrase=passphrase)
|
|
1205
|
-
|
|
1206
|
-
def restore_encrypted_archive(
|
|
1207
|
-
self,
|
|
1208
|
-
archive_path,
|
|
1209
|
-
*,
|
|
1210
|
-
passphrase: str,
|
|
1211
|
-
dry_run: bool = False,
|
|
1212
|
-
confirm: bool = False,
|
|
1213
|
-
) -> Dict[str, Any]:
|
|
1214
|
-
self._require()
|
|
1215
|
-
archive = EncryptedBrainArchive(
|
|
1216
|
-
BrainArchivePaths(
|
|
1217
|
-
db_path=Path(self._kg.db_path),
|
|
1218
|
-
blob_dir=Path(self._kg.blob_dir),
|
|
1219
|
-
data_dir=self._data_dir,
|
|
1220
|
-
)
|
|
1221
|
-
)
|
|
1222
|
-
return archive.restore(
|
|
1223
|
-
Path(archive_path),
|
|
1224
|
-
passphrase=passphrase,
|
|
1225
|
-
target=BrainArchivePaths(
|
|
1226
|
-
db_path=Path(self._kg.db_path),
|
|
1227
|
-
blob_dir=Path(self._kg.blob_dir),
|
|
1228
|
-
data_dir=self._data_dir,
|
|
1229
|
-
),
|
|
1230
|
-
dry_run=dry_run,
|
|
1231
|
-
confirm=confirm,
|
|
1232
|
-
)
|
|
1233
|
-
|
|
1234
|
-
def import_encrypted_archive(
|
|
1235
|
-
self,
|
|
1236
|
-
archive_path,
|
|
1237
|
-
*,
|
|
1238
|
-
passphrase: str,
|
|
1239
|
-
dry_run: bool = False,
|
|
1240
|
-
confirm: bool = False,
|
|
1241
|
-
) -> Dict[str, Any]:
|
|
1242
|
-
result = self.restore_encrypted_archive(
|
|
1243
|
-
archive_path,
|
|
1244
|
-
passphrase=passphrase,
|
|
1245
|
-
dry_run=dry_run,
|
|
1246
|
-
confirm=confirm,
|
|
1247
|
-
)
|
|
1248
|
-
result["operation"] = "import"
|
|
1249
|
-
return result
|
|
1250
|
-
|
|
1251
|
-
# ── status surface ───────────────────────────────────────────────────────
|
|
1252
|
-
def snapshot_metadata(self) -> Dict[str, Any]:
|
|
1253
|
-
if not self.available():
|
|
1254
|
-
return {"available": False}
|
|
1255
|
-
return {
|
|
1256
|
-
"available": True,
|
|
1257
|
-
**self._kg.schema_versions(),
|
|
1258
|
-
"stats": self._kg.stats(),
|
|
1259
|
-
"provenance": self._kg.provenance_stats(),
|
|
1260
|
-
"storage": (
|
|
1261
|
-
self._kg.storage_engine.capabilities().as_dict()
|
|
1262
|
-
if getattr(self._kg, "storage_engine", None) is not None
|
|
1263
|
-
else {"engine": "sqlite", "available": True}
|
|
1264
|
-
),
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
def storage_status(self) -> Dict[str, Any]:
|
|
1268
|
-
if not self.available():
|
|
1269
|
-
return {"available": False}
|
|
1270
|
-
return {
|
|
1271
|
-
"available": True,
|
|
1272
|
-
"active": (
|
|
1273
|
-
self._kg.storage_engine.capabilities().as_dict()
|
|
1274
|
-
if getattr(self._kg, "storage_engine", None) is not None
|
|
1275
|
-
else {"engine": "sqlite", "available": True}
|
|
1276
|
-
),
|
|
1277
|
-
"postgres": PostgresEngine("", schema="lattice_brain").capabilities().as_dict(),
|
|
1278
|
-
"backup_health": self.backup_health(),
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
def backup_health(self) -> Dict[str, Any]:
|
|
1282
|
-
self._exports_dir.mkdir(parents=True, exist_ok=True)
|
|
1283
|
-
backups = sorted(
|
|
1284
|
-
[
|
|
1285
|
-
p for p in self._exports_dir.glob("*")
|
|
1286
|
-
if p.is_file() and (p.suffix == ".zip" or p.suffix == ".latticebrain")
|
|
1287
|
-
],
|
|
1288
|
-
key=lambda p: p.stat().st_mtime,
|
|
1289
|
-
reverse=True,
|
|
1290
|
-
)
|
|
1291
|
-
latest = backups[0] if backups else None
|
|
1292
|
-
return {
|
|
1293
|
-
"available": True,
|
|
1294
|
-
"directory": str(self._exports_dir),
|
|
1295
|
-
"count": len(backups),
|
|
1296
|
-
"latest": str(latest) if latest else None,
|
|
1297
|
-
"latest_bytes": latest.stat().st_size if latest else 0,
|
|
1298
|
-
"encrypted_archives": sum(1 for p in backups if p.suffix == ".latticebrain"),
|
|
1299
|
-
"zip_backups": sum(1 for p in backups if p.suffix == ".zip"),
|
|
1300
|
-
}
|
|
1301
|
-
|
|
1302
|
-
def postgres_docker_setup(
|
|
1303
|
-
self,
|
|
1304
|
-
*,
|
|
1305
|
-
consent: bool,
|
|
1306
|
-
dry_run: bool = False,
|
|
1307
|
-
port: int = 5432,
|
|
1308
|
-
) -> Dict[str, Any]:
|
|
1309
|
-
wizard = DockerPostgresWizard(self._data_dir / "postgres", port=port)
|
|
1310
|
-
return wizard.start(consent=consent, dry_run=dry_run)
|
|
1311
|
-
|
|
1312
|
-
def migrate_sqlite_to_postgres(
|
|
1313
|
-
self,
|
|
1314
|
-
*,
|
|
1315
|
-
dsn: str,
|
|
1316
|
-
schema: str = "lattice_brain",
|
|
1317
|
-
dry_run: bool = True,
|
|
1318
|
-
) -> Dict[str, Any]:
|
|
1319
|
-
self._require()
|
|
1320
|
-
if not dsn:
|
|
1321
|
-
raise ValueError("Postgres DSN is required for SQLite to Postgres migration.")
|
|
1322
|
-
migrator = SQLiteToPostgresMigrator(
|
|
1323
|
-
Path(self._kg.db_path),
|
|
1324
|
-
PostgresEngine(dsn, schema=schema),
|
|
1325
|
-
)
|
|
1326
|
-
if dry_run:
|
|
1327
|
-
return migrator.migrate(dry_run=True)
|
|
1328
|
-
backup = self.backup()
|
|
1329
|
-
verification = self.verify_backup(backup["path"])
|
|
1330
|
-
if not verification.get("ok"):
|
|
1331
|
-
raise RuntimeError(
|
|
1332
|
-
"Pre-migration backup verification failed; Postgres migration was not started: "
|
|
1333
|
-
+ "; ".join(verification.get("errors") or [])
|
|
1334
|
-
)
|
|
1335
|
-
result = migrator.migrate(dry_run=False)
|
|
1336
|
-
result["pre_migration_backup"] = {
|
|
1337
|
-
"path": backup["path"],
|
|
1338
|
-
"verified": True,
|
|
1339
|
-
"manifest": backup.get("manifest"),
|
|
1340
|
-
}
|
|
1341
|
-
return result
|
|
1342
|
-
|
|
1343
|
-
def recent_ingestions(self, *, limit: int = 50, source_type: Optional[str] = None) -> Dict[str, Any]:
|
|
1344
|
-
"""Recent provenance records (newest first) for the ingestion-sources UI."""
|
|
1345
|
-
if not self.available():
|
|
1346
|
-
return {"items": [], "count": 0}
|
|
1347
|
-
return self._kg.list_provenance(limit=limit, source_type=source_type)
|