ltcai 11.5.2 → 11.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -148
- package/bin/ltcai.js +234 -24
- package/docs/CHANGELOG.md +119 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +8 -6
- package/docs/ENTERPRISE.md +2 -1
- package/docs/MULTI_AGENT_RUNTIME.md +12 -5
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +6 -3
- package/docs/REALTIME_COLLABORATION.md +1 -1
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/WORKFLOW_DESIGNER.md +3 -2
- package/docs/kg-schema.md +1 -1
- package/docs/v11.6.0_ONE_DOOR_PLAN.md +170 -0
- package/lattice_brain/__init__.py +42 -79
- package/lattice_brain/graph/__init__.py +9 -24
- package/lattice_brain/graph/_kg_common/__init__.py +13 -24
- package/lattice_brain/ingestion/__init__.py +19 -58
- package/lattice_brain/ingestion/pipeline.py +20 -398
- package/lattice_brain/multimodal/__init__.py +7 -18
- package/lattice_brain/multimodal/images.py +6 -264
- package/lattice_brain/multimodal/video.py +8 -247
- package/lattice_brain/runtime/__init__.py +8 -79
- package/lattice_brain/runtime/hooks.py +22 -584
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/agent_worker_seam.py +7 -80
- package/latticeai/api/health.py +6 -20
- package/latticeai/api/local_files.py +16 -613
- package/latticeai/api/models.py +12 -87
- package/latticeai/api/search.py +17 -255
- package/latticeai/api/tools.py +69 -750
- package/latticeai/api/voice_capture.py +8 -70
- package/latticeai/api/worker_compute.py +842 -0
- package/latticeai/api/worker_seams.py +216 -0
- package/latticeai/app_factory.py +23 -232
- package/latticeai/cli/entrypoint.py +36 -249
- package/latticeai/core/agent_permission.py +16 -91
- package/latticeai/core/messages.py +89 -531
- package/latticeai/runtime/access_runtime.py +0 -14
- package/latticeai/runtime/bootstrap.py +10 -21
- package/latticeai/runtime/brain_runtime.py +35 -43
- package/latticeai/runtime/build_phases/__init__.py +21 -37
- package/latticeai/runtime/build_phases/features.py +99 -389
- package/latticeai/runtime/build_phases/foundation.py +82 -392
- package/latticeai/runtime/build_phases/web.py +91 -408
- package/latticeai/runtime/build_phases/worker_profile.py +244 -0
- package/latticeai/runtime/lifespan_runtime.py +7 -16
- package/latticeai/runtime/platform_services_runtime.py +1 -15
- package/latticeai/runtime/runtime_context.py +13 -130
- package/latticeai/runtime/security_runtime.py +10 -103
- package/latticeai/services/architecture_readiness.py +82 -43
- package/latticeai/services/model_runtime/__init__.py +2 -11
- package/latticeai/services/model_runtime/service.py +4 -28
- package/latticeai/services/p_reinforce.py +10 -261
- package/latticeai/services/product_readiness.py +31 -38
- package/latticeai/services/search_service.py +37 -795
- package/latticeai/services/tool_dispatch.py +30 -386
- package/latticeai/services/voice_capture.py +13 -107
- package/latticeai/tools/__init__.py +22 -49
- package/latticeai/tools/commands.py +0 -163
- package/latticeai/tools/computer.py +0 -39
- package/latticeai/tools/documents.py +1 -134
- package/latticeai/tools/filesystem.py +1 -247
- package/latticeai/tools/knowledge.py +1 -52
- package/latticeai/tools/local_files.py +0 -20
- package/latticeai/worker_app.py +75 -0
- package/package.json +2 -3
- package/requirements.txt +0 -5
- package/scripts/agent_eval.py +16 -28
- package/scripts/brain_quality_eval.py +20 -183
- package/scripts/bump_version.py +5 -5
- package/scripts/check_current_release_docs.mjs +7 -4
- package/scripts/check_openapi_drift.mjs +10 -0
- package/scripts/check_server_i18n.mjs +2 -18
- package/scripts/compose_openapi.py +377 -0
- package/scripts/export_openapi.py +32 -4
- package/scripts/gen_messages_catalog_fixture.py +302 -0
- package/scripts/gen_openapi_fragments.py +365 -0
- package/scripts/gen_redact_fixture.py +196 -0
- package/scripts/gen_worker_allowlist_fixture.py +115 -0
- package/scripts/generate_agent_parity_fixtures.py +33 -14
- package/scripts/openapi_route_families.json +2161 -0
- package/scripts/release_screen_claims.json +54 -0
- package/scripts/run_integration_tests.mjs +134 -29
- package/scripts/run_sidecar_e2e.mjs +104 -11
- package/scripts/wheel_smoke.py +32 -22
- package/src-tauri/Cargo.lock +375 -9
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/src/backend.rs +151 -45
- package/src-tauri/src/main.rs +18 -18
- package/src-tauri/src/topology.rs +58 -88
- package/src-tauri/tauri.conf.json +1 -2
- package/static/app/asset-manifest.json +41 -41
- package/static/app/assets/{Act-DcQizkl1.js → Act-BPcVAbOL.js} +1 -1
- package/static/app/assets/AdminConsole-Bw1ATQL0.js +1 -0
- package/static/app/assets/{Brain-3VCSHFcn.js → Brain-CT92Kos0.js} +2 -2
- package/static/app/assets/{BrainHome-Qm8eaztx.js → BrainHome-CFBkt1K_.js} +1 -1
- package/static/app/assets/{BrainSignals-DS9BtKOW.js → BrainSignals-ReLWF2H8.js} +1 -1
- package/static/app/assets/Capture-BsTokYkk.js +1 -0
- package/static/app/assets/{Chronicle-BGvuAchH.js → Chronicle-B6f0T9id.js} +1 -1
- package/static/app/assets/{CommandPalette-Bqhm0Urn.js → CommandPalette-CuvjTv1u.js} +1 -1
- package/static/app/assets/{Library-BV6NnF0a.js → Library-BGJbG9Hd.js} +1 -1
- package/static/app/assets/{LivingBrain-GzenJchP.js → LivingBrain-DGYK_Jsa.js} +1 -1
- package/static/app/assets/{ProductFlow-DEP6-vML.js → ProductFlow-DXBC6brE.js} +1 -1
- package/static/app/assets/{ReviewCard-CNZ7XjWG.js → ReviewCard-HXRle3qq.js} +2 -2
- package/static/app/assets/System-CMHSO9qM.js +1 -0
- package/static/app/assets/arrow-left-BfmkskWx.js +1 -0
- package/static/app/assets/{bot-B_K1Tdmw.js → bot-Cn8bWRuq.js} +1 -1
- package/static/app/assets/{brain-DWyaV1L1.js → brain-CQJberbE.js} +1 -1
- package/static/app/assets/{button-aTn4s84A.js → button-Ct9f2_oT.js} +1 -1
- package/static/app/assets/circle-check-DruOxB-4.js +1 -0
- package/static/app/assets/{circle-pause-xKgeGXkT.js → circle-pause-CmzC_apg.js} +1 -1
- package/static/app/assets/{circle-play-DkT6tYPX.js → circle-play-D8mW2aQ7.js} +1 -1
- package/static/app/assets/{cpu-85xYObUC.js → cpu-DZcdd0PZ.js} +1 -1
- package/static/app/assets/{download-B5Fm7YXo.js → download-bv1KEPGQ.js} +1 -1
- package/static/app/assets/{folder-open-kk2Xa52u.js → folder-open-d-Pip5gr.js} +1 -1
- package/static/app/assets/{hard-drive-DkA3zBW_.js → hard-drive-D20iavUb.js} +1 -1
- package/static/app/assets/index-D9x-kSNy.css +2 -0
- package/static/app/assets/{index-BMPdTmlY.js → index-Do83hDzJ.js} +3 -3
- package/static/app/assets/{input-B0nRf2jO.js → input-BLXVNmj1.js} +1 -1
- package/static/app/assets/{link-2-Dwb4gnTc.js → link-2-BPJOFlAy.js} +1 -1
- package/static/app/assets/{permissionCopy-CQDUBrOZ.js → permissionCopy-ChdJd493.js} +1 -1
- package/static/app/assets/primitives-Cv5tbZBY.js +1 -0
- package/static/app/assets/search-CT9aho2j.js +1 -0
- package/static/app/assets/{share-2-BsrxFglO.js → share-2-YNX_NtMU.js} +1 -1
- package/static/app/assets/{shield-alert-5BStfp2_.js → shield-alert-DuQ3zrVL.js} +1 -1
- package/static/app/assets/{textarea-Cg8IUA-k.js → textarea-DqwLnli4.js} +1 -1
- package/static/app/assets/{useFocusTrap-CYKvE46M.js → useFocusTrap-ZVI98jaW.js} +1 -1
- package/static/app/assets/{useMutation-CSn9t1op.js → useMutation-CVC4qv_D.js} +1 -1
- package/static/app/assets/{useQuery-CY2OI2uy.js → useQuery-C7BeG4HU.js} +1 -1
- package/static/app/assets/{utils-Ddol2RWD.js → utils-CiFtIdZq.js} +1 -1
- package/static/app/assets/{workspace-BqDwOz_p.js → workspace-DQz9vIId.js} +1 -1
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/lattice_brain/archive.py +0 -522
- package/lattice_brain/context.py +0 -325
- package/lattice_brain/conversations.py +0 -382
- package/lattice_brain/core.py +0 -82
- package/lattice_brain/graph/_kg_contract.py +0 -249
- package/lattice_brain/graph/curator.py +0 -676
- package/lattice_brain/graph/discovery.py +0 -595
- package/lattice_brain/graph/discovery_index/__init__.py +0 -35
- package/lattice_brain/graph/discovery_index/cleanup.py +0 -182
- package/lattice_brain/graph/discovery_index/extract.py +0 -137
- package/lattice_brain/graph/discovery_index/scan.py +0 -411
- package/lattice_brain/graph/discovery_index/upsert.py +0 -495
- package/lattice_brain/graph/documents.py +0 -380
- package/lattice_brain/graph/fusion.py +0 -395
- package/lattice_brain/graph/identity.py +0 -175
- package/lattice_brain/graph/image_vectors.py +0 -230
- package/lattice_brain/graph/ingest.py +0 -829
- package/lattice_brain/graph/network.py +0 -205
- package/lattice_brain/graph/proactive.py +0 -724
- package/lattice_brain/graph/projection/__init__.py +0 -42
- package/lattice_brain/graph/projection/curation.py +0 -500
- package/lattice_brain/graph/projection/v2_schema.py +0 -518
- package/lattice_brain/graph/provenance.py +0 -524
- package/lattice_brain/graph/rerank.py +0 -163
- package/lattice_brain/graph/retrieval/__init__.py +0 -54
- package/lattice_brain/graph/retrieval/context.py +0 -197
- package/lattice_brain/graph/retrieval/graph_view.py +0 -319
- package/lattice_brain/graph/retrieval/hybrid.py +0 -488
- package/lattice_brain/graph/retrieval/maintenance.py +0 -121
- package/lattice_brain/graph/retrieval/signals.py +0 -95
- package/lattice_brain/graph/retrieval_docgen.py +0 -253
- package/lattice_brain/graph/retrieval_policy.py +0 -180
- package/lattice_brain/graph/retrieval_reads.py +0 -769
- package/lattice_brain/graph/retrieval_vector/__init__.py +0 -42
- package/lattice_brain/graph/retrieval_vector/fingerprint.py +0 -97
- package/lattice_brain/graph/retrieval_vector/indexing.py +0 -347
- package/lattice_brain/graph/retrieval_vector/search.py +0 -560
- package/lattice_brain/graph/retrieval_vector/status.py +0 -374
- package/lattice_brain/graph/schema.py +0 -792
- package/lattice_brain/graph/store.py +0 -268
- package/lattice_brain/graph/vector_index/__init__.py +0 -85
- package/lattice_brain/graph/vector_index/base.py +0 -167
- package/lattice_brain/graph/vector_index/brute_force.py +0 -110
- package/lattice_brain/graph/vector_index/hnsw.py +0 -290
- package/lattice_brain/graph/vector_index/jobs.py +0 -287
- package/lattice_brain/graph/vector_index/quantized.py +0 -148
- package/lattice_brain/graph/vector_index/selector.py +0 -161
- package/lattice_brain/graph/write_master.py +0 -308
- package/lattice_brain/ingestion/_contract.py +0 -90
- package/lattice_brain/ingestion/folder_scan.py +0 -57
- package/lattice_brain/ingestion/folders.py +0 -258
- package/lattice_brain/ingestion/jobs_api.py +0 -107
- package/lattice_brain/ingestion/routing.py +0 -295
- package/lattice_brain/ingestion_jobs.py +0 -380
- package/lattice_brain/memory.py +0 -75
- package/lattice_brain/portability/__init__.py +0 -90
- package/lattice_brain/portability/_contract.py +0 -42
- package/lattice_brain/portability/backups.py +0 -338
- package/lattice_brain/portability/bundles.py +0 -136
- package/lattice_brain/portability/constants.py +0 -93
- package/lattice_brain/portability/fsops.py +0 -138
- package/lattice_brain/portability/service.py +0 -41
- package/lattice_brain/portability/sharing.py +0 -710
- package/lattice_brain/quality.py +0 -543
- package/lattice_brain/retrieval_benchmark_fixtures.py +0 -92
- package/lattice_brain/runtime/agent_runtime.py +0 -859
- package/lattice_brain/runtime/contracts.py +0 -460
- package/lattice_brain/runtime/multi_agent.py +0 -942
- package/lattice_brain/runtime/statuses.py +0 -10
- package/lattice_brain/sealed_box.py +0 -240
- package/lattice_brain/self_model.py +0 -675
- package/lattice_brain/sensitivity.py +0 -94
- package/lattice_brain/storage/__init__.py +0 -22
- package/lattice_brain/storage/base.py +0 -100
- package/lattice_brain/storage/docker.py +0 -105
- package/lattice_brain/storage/factory.py +0 -31
- package/lattice_brain/storage/migration.py +0 -191
- package/lattice_brain/storage/postgres.py +0 -123
- package/lattice_brain/storage/sqlite.py +0 -143
- package/lattice_brain/synthesis.py +0 -824
- package/lattice_brain/workflow.py +0 -497
- package/latticeai/api/admin.py +0 -471
- package/latticeai/api/agent_registry.py +0 -105
- package/latticeai/api/agents.py +0 -228
- package/latticeai/api/auth.py +0 -383
- package/latticeai/api/automation_intelligence.py +0 -401
- package/latticeai/api/brain_intelligence.py +0 -199
- package/latticeai/api/browser.py +0 -493
- package/latticeai/api/change_proposals.py +0 -89
- package/latticeai/api/chat.py +0 -572
- package/latticeai/api/chat_agent_http.py +0 -892
- package/latticeai/api/chat_contracts.py +0 -73
- package/latticeai/api/chat_documents.py +0 -276
- package/latticeai/api/chat_helpers.py +0 -460
- package/latticeai/api/chat_history.py +0 -101
- package/latticeai/api/chat_hybrid.py +0 -113
- package/latticeai/api/chat_intents.py +0 -646
- package/latticeai/api/chat_stream.py +0 -216
- package/latticeai/api/chronicle.py +0 -63
- package/latticeai/api/command_center.py +0 -51
- package/latticeai/api/computer_use.py +0 -474
- package/latticeai/api/evidence_actions.py +0 -48
- package/latticeai/api/features.py +0 -70
- package/latticeai/api/funnel_metrics.py +0 -31
- package/latticeai/api/garden.py +0 -34
- package/latticeai/api/hooks.py +0 -165
- package/latticeai/api/index_jobs.py +0 -145
- package/latticeai/api/invitations.py +0 -100
- package/latticeai/api/knowledge_graph.py +0 -536
- package/latticeai/api/marketplace.py +0 -105
- package/latticeai/api/mcp.py +0 -482
- package/latticeai/api/memory.py +0 -270
- package/latticeai/api/network.py +0 -81
- package/latticeai/api/network_boundary.py +0 -225
- package/latticeai/api/permission_mode.py +0 -61
- package/latticeai/api/permissions.py +0 -436
- package/latticeai/api/plugins.py +0 -126
- package/latticeai/api/portability.py +0 -391
- package/latticeai/api/project_sessions.py +0 -114
- package/latticeai/api/realtime.py +0 -118
- package/latticeai/api/review_queue.py +0 -364
- package/latticeai/api/security_dashboard.py +0 -604
- package/latticeai/api/setup.py +0 -319
- package/latticeai/api/static_routes.py +0 -354
- package/latticeai/api/ui_redirects.py +0 -26
- package/latticeai/api/workflow_designer.py +0 -394
- package/latticeai/api/workspace.py +0 -856
- package/latticeai/api/workspace_scope.py +0 -125
- package/latticeai/core/agent/__init__.py +0 -93
- package/latticeai/core/agent/_contract.py +0 -79
- package/latticeai/core/agent/context.py +0 -57
- package/latticeai/core/agent/deps.py +0 -125
- package/latticeai/core/agent/execution.py +0 -622
- package/latticeai/core/agent/planning.py +0 -145
- package/latticeai/core/agent/recovery.py +0 -157
- package/latticeai/core/agent/runtime.py +0 -210
- package/latticeai/core/agent/verification.py +0 -231
- package/latticeai/core/agent_eval.py +0 -739
- package/latticeai/core/agent_helpers.py +0 -493
- package/latticeai/core/agent_profiles.py +0 -110
- package/latticeai/core/agent_prompts.py +0 -171
- package/latticeai/core/agent_registry.py +0 -232
- package/latticeai/core/agent_state.py +0 -41
- package/latticeai/core/agent_trace.py +0 -104
- package/latticeai/core/artifact_ledger.py +0 -109
- package/latticeai/core/audit.py +0 -260
- package/latticeai/core/builtin_hooks.py +0 -105
- package/latticeai/core/context_builder.py +0 -394
- package/latticeai/core/document_generator.py +0 -103
- package/latticeai/core/enterprise.py +0 -154
- package/latticeai/core/enterprise_admin.py +0 -158
- package/latticeai/core/file_generation/__init__.py +0 -115
- package/latticeai/core/file_generation/bundles.py +0 -76
- package/latticeai/core/file_generation/extraction.py +0 -154
- package/latticeai/core/file_generation/inference.py +0 -235
- package/latticeai/core/file_generation/orchestration.py +0 -152
- package/latticeai/core/file_generation/prompting.py +0 -117
- package/latticeai/core/file_generation/repair.py +0 -114
- package/latticeai/core/file_generation/sanitize.py +0 -61
- package/latticeai/core/file_generation/validation.py +0 -201
- package/latticeai/core/invitations.py +0 -132
- package/latticeai/core/legacy_compatibility.py +0 -243
- package/latticeai/core/logging_safety.py +0 -46
- package/latticeai/core/marketplace.py +0 -293
- package/latticeai/core/mcp_catalog.py +0 -452
- package/latticeai/core/mcp_registry.py +0 -506
- package/latticeai/core/network_boundary.py +0 -168
- package/latticeai/core/oidc.py +0 -208
- package/latticeai/core/plugins.py +0 -432
- package/latticeai/core/product_hardening.py +0 -218
- package/latticeai/core/project_sessions.py +0 -337
- package/latticeai/core/realtime.py +0 -238
- package/latticeai/core/run_explain.py +0 -426
- package/latticeai/core/run_store.py +0 -252
- package/latticeai/core/timezones.py +0 -80
- package/latticeai/core/workspace_computer_memory.py +0 -84
- package/latticeai/core/workspace_graph_trace.py +0 -155
- package/latticeai/core/workspace_indexing.py +0 -102
- package/latticeai/core/workspace_memory.py +0 -77
- package/latticeai/core/workspace_onboarding.py +0 -104
- package/latticeai/core/workspace_os.py +0 -978
- package/latticeai/core/workspace_os_constants.py +0 -126
- package/latticeai/core/workspace_os_state.py +0 -180
- package/latticeai/core/workspace_os_utils.py +0 -103
- package/latticeai/core/workspace_permissions.py +0 -101
- package/latticeai/core/workspace_plugins.py +0 -97
- package/latticeai/core/workspace_relationships.py +0 -99
- package/latticeai/core/workspace_reorganization.py +0 -335
- package/latticeai/core/workspace_review_items.py +0 -112
- package/latticeai/core/workspace_runs.py +0 -726
- package/latticeai/core/workspace_skills.py +0 -109
- package/latticeai/core/workspace_snapshots.py +0 -198
- package/latticeai/core/workspace_timeline.py +0 -110
- package/latticeai/integrations/__init__.py +0 -0
- package/latticeai/integrations/telegram_bot/__init__.py +0 -123
- package/latticeai/integrations/telegram_bot/__main__.py +0 -17
- package/latticeai/integrations/telegram_bot/config.py +0 -86
- package/latticeai/integrations/telegram_bot/dispatch.py +0 -311
- package/latticeai/integrations/telegram_bot/flows.py +0 -478
- package/latticeai/integrations/telegram_bot/helpers.py +0 -322
- package/latticeai/integrations/telegram_bot/screens.py +0 -394
- package/latticeai/runtime/audit_runtime.py +0 -76
- package/latticeai/runtime/automation_runtime.py +0 -81
- package/latticeai/runtime/chat_wiring.py +0 -141
- package/latticeai/runtime/context_runtime.py +0 -66
- package/latticeai/runtime/feature_toggle_wiring.py +0 -157
- package/latticeai/runtime/history_runtime.py +0 -163
- package/latticeai/runtime/history_writer.py +0 -138
- package/latticeai/runtime/hooks_runtime.py +0 -77
- package/latticeai/runtime/model_wiring.py +0 -68
- package/latticeai/runtime/namespace_runtime.py +0 -163
- package/latticeai/runtime/network_boundary_wiring.py +0 -117
- package/latticeai/runtime/network_config_runtime.py +0 -56
- package/latticeai/runtime/permission_mode_wiring.py +0 -112
- package/latticeai/runtime/persistence_runtime.py +0 -159
- package/latticeai/runtime/platform_runtime_wiring.py +0 -89
- package/latticeai/runtime/review_wiring.py +0 -42
- package/latticeai/runtime/router_registration.py +0 -693
- package/latticeai/runtime/service_singletons.py +0 -55
- package/latticeai/runtime/sso_config_runtime.py +0 -128
- package/latticeai/runtime/user_key_runtime.py +0 -106
- package/latticeai/runtime/web_runtime.py +0 -92
- package/latticeai/server_app.py +0 -51
- package/latticeai/services/app_context.py +0 -130
- package/latticeai/services/automation_execution.py +0 -266
- package/latticeai/services/automation_intelligence.py +0 -614
- package/latticeai/services/brain_automation.py +0 -191
- package/latticeai/services/brain_intelligence/__init__.py +0 -58
- package/latticeai/services/brain_intelligence/_contract.py +0 -71
- package/latticeai/services/brain_intelligence/consistency.py +0 -193
- package/latticeai/services/brain_intelligence/constants.py +0 -47
- package/latticeai/services/brain_intelligence/digest.py +0 -258
- package/latticeai/services/brain_intelligence/health.py +0 -331
- package/latticeai/services/brain_intelligence/proposals.py +0 -259
- package/latticeai/services/brain_intelligence/sampling.py +0 -84
- package/latticeai/services/brain_intelligence/service.py +0 -48
- package/latticeai/services/change_proposals.py +0 -471
- package/latticeai/services/chat_service.py +0 -243
- package/latticeai/services/chronicle.py +0 -555
- package/latticeai/services/cloud_egress_audit.py +0 -85
- package/latticeai/services/cloud_extraction.py +0 -129
- package/latticeai/services/cloud_streaming.py +0 -268
- package/latticeai/services/cloud_token_guard.py +0 -84
- package/latticeai/services/command_center.py +0 -548
- package/latticeai/services/evidence_actions.py +0 -258
- package/latticeai/services/feature_toggles.py +0 -502
- package/latticeai/services/folder_watch.py +0 -520
- package/latticeai/services/funnel_metrics.py +0 -307
- package/latticeai/services/hybrid_chat.py +0 -316
- package/latticeai/services/hybrid_context.py +0 -228
- package/latticeai/services/hybrid_policy.py +0 -129
- package/latticeai/services/interop_bridges.py +0 -978
- package/latticeai/services/local_knowledge.py +0 -465
- package/latticeai/services/memory_service/__init__.py +0 -52
- package/latticeai/services/memory_service/_contract.py +0 -100
- package/latticeai/services/memory_service/brief.py +0 -431
- package/latticeai/services/memory_service/constants.py +0 -57
- package/latticeai/services/memory_service/maintenance.py +0 -138
- package/latticeai/services/memory_service/manager.py +0 -186
- package/latticeai/services/memory_service/proof.py +0 -136
- package/latticeai/services/memory_service/recall.py +0 -225
- package/latticeai/services/memory_service/service.py +0 -48
- package/latticeai/services/memory_service/stores.py +0 -110
- package/latticeai/services/mode_store.py +0 -132
- package/latticeai/services/model_recommendation.py +0 -224
- package/latticeai/services/model_runtime/cloud.py +0 -87
- package/latticeai/services/network_boundary_service.py +0 -117
- package/latticeai/services/obsidian_bridge.py +0 -609
- package/latticeai/services/openai_compatible_adapter.py +0 -101
- package/latticeai/services/permission_mode_service.py +0 -122
- package/latticeai/services/platform_runtime.py +0 -366
- package/latticeai/services/review_queue.py +0 -380
- package/latticeai/services/router_context.py +0 -59
- package/latticeai/services/run_executor.py +0 -387
- package/latticeai/services/self_model_service.py +0 -171
- package/latticeai/services/setup_detection.py +0 -147
- package/latticeai/services/triggers.py +0 -378
- package/latticeai/services/upload_service.py +0 -172
- package/latticeai/services/workspace_service.py +0 -165
- package/latticeai/setup/__init__.py +0 -25
- package/latticeai/setup/auto_setup.py +0 -846
- package/latticeai/setup/demo_corpus.py +0 -98
- package/latticeai/setup/wizard/__init__.py +0 -126
- package/latticeai/setup/wizard/catalog.py +0 -175
- package/latticeai/setup/wizard/detect.py +0 -302
- package/latticeai/setup/wizard/install.py +0 -348
- package/latticeai/setup/wizard/paths.py +0 -165
- package/latticeai/setup/wizard/plans.py +0 -74
- package/latticeai/setup/wizard/recommend.py +0 -320
- package/scripts/bench_agent_smoke.py +0 -409
- package/scripts/bench_models.py +0 -540
- package/scripts/bench_vector_index.py +0 -295
- package/scripts/funnel_soft_gate.py +0 -192
- package/scripts/generate_agent_loop_fixtures.py +0 -994
- package/scripts/generate_rust_parity_fixtures.py +0 -908
- package/scripts/migrate_brain_storage.py +0 -57
- package/scripts/parity_fixture_corpus_context.py +0 -162
- package/scripts/parity_fixture_corpus_docgen.py +0 -341
- package/scripts/profile_kg.py +0 -355
- package/server.py +0 -30
- package/static/app/assets/AdminConsole-cf4npybT.js +0 -1
- package/static/app/assets/Capture-DiQ219jW.js +0 -1
- package/static/app/assets/System-CieofHQa.js +0 -1
- package/static/app/assets/arrow-left-kfsrk0mv.js +0 -1
- package/static/app/assets/circle-check-qqLug9nU.js +0 -1
- package/static/app/assets/index-DxmOfNRi.css +0 -2
- package/static/app/assets/primitives-SNp0LRJz.js +0 -1
- package/static/app/assets/search-BcHqkjoy.js +0 -1
|
@@ -1,268 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
# ruff: noqa: F403,F405
|
|
4
|
-
from ._kg_common import * # noqa: F403,F401
|
|
5
|
-
from .discovery import KnowledgeGraphDiscoveryMixin
|
|
6
|
-
from .discovery_index import KnowledgeGraphLocalIndexMixin
|
|
7
|
-
from .documents import KnowledgeGraphDocumentsMixin
|
|
8
|
-
from .ingest import KnowledgeGraphIngestMixin
|
|
9
|
-
from .projection import KnowledgeGraphProjectionMixin
|
|
10
|
-
from .provenance import KnowledgeGraphProvenanceMixin
|
|
11
|
-
from .retrieval import KnowledgeGraphRetrievalMixin
|
|
12
|
-
from .retrieval_docgen import KnowledgeGraphDocGenMixin
|
|
13
|
-
from .retrieval_reads import KnowledgeGraphReadsMixin
|
|
14
|
-
from .retrieval_vector import KnowledgeGraphVectorMixin
|
|
15
|
-
from .write_master import KnowledgeGraphWriteMixin
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class KnowledgeGraphStore(
|
|
19
|
-
KnowledgeGraphProjectionMixin,
|
|
20
|
-
KnowledgeGraphWriteMixin,
|
|
21
|
-
KnowledgeGraphDiscoveryMixin,
|
|
22
|
-
KnowledgeGraphLocalIndexMixin,
|
|
23
|
-
KnowledgeGraphIngestMixin,
|
|
24
|
-
KnowledgeGraphProvenanceMixin,
|
|
25
|
-
KnowledgeGraphDocumentsMixin,
|
|
26
|
-
KnowledgeGraphRetrievalMixin,
|
|
27
|
-
KnowledgeGraphReadsMixin,
|
|
28
|
-
KnowledgeGraphVectorMixin,
|
|
29
|
-
KnowledgeGraphDocGenMixin,
|
|
30
|
-
):
|
|
31
|
-
def __init__(
|
|
32
|
-
self,
|
|
33
|
-
db_path: Path,
|
|
34
|
-
blob_dir: Path,
|
|
35
|
-
embedder: Any = None,
|
|
36
|
-
storage_engine: Any = None,
|
|
37
|
-
):
|
|
38
|
-
self.db_path = Path(db_path)
|
|
39
|
-
self.blob_dir = Path(blob_dir)
|
|
40
|
-
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
41
|
-
self.blob_dir.mkdir(parents=True, exist_ok=True)
|
|
42
|
-
if storage_engine is None:
|
|
43
|
-
from ..storage import SQLiteEngine
|
|
44
|
-
|
|
45
|
-
storage_engine = SQLiteEngine(self.db_path)
|
|
46
|
-
storage_caps = storage_engine.capabilities()
|
|
47
|
-
if not storage_caps.available:
|
|
48
|
-
raise RuntimeError(storage_caps.reason or "Brain storage is unavailable.")
|
|
49
|
-
if storage_caps.engine != "sqlite":
|
|
50
|
-
raise RuntimeError(
|
|
51
|
-
"KnowledgeGraphStore currently requires SQLiteEngine. "
|
|
52
|
-
"Explicit non-SQLite storage must use the migration/scale tooling; "
|
|
53
|
-
"no SQLite fallback is attempted."
|
|
54
|
-
)
|
|
55
|
-
self.storage_engine = storage_engine
|
|
56
|
-
# The embedder is swappable behind a fixed interface
|
|
57
|
-
# (model_id/dim/embed/encode/decode/similarity). Defaults to the
|
|
58
|
-
# deterministic, offline hash model so the store works with no config;
|
|
59
|
-
# server_app injects a provider-backed embedder from Config.
|
|
60
|
-
self._embedding_model = (
|
|
61
|
-
embedder if embedder is not None else LocalEmbeddingModel()
|
|
62
|
-
)
|
|
63
|
-
self._v2_projection_available = False
|
|
64
|
-
self._init_db()
|
|
65
|
-
# Read graph queries from the v2 projection (kgv2_* views) when available.
|
|
66
|
-
# Toggle off (e.g. in tests) to compare against the legacy tables.
|
|
67
|
-
self._read_from_v2 = (
|
|
68
|
-
KGStoreV2 is not None
|
|
69
|
-
and _READ_FROM_V2_DEFAULT
|
|
70
|
-
and self._v2_projection_available
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
def _read_tables(self) -> tuple:
|
|
74
|
-
"""Return (nodes_table, edges_table) for read queries.
|
|
75
|
-
|
|
76
|
-
Same read code runs against the legacy tables or the v2 reconstruction
|
|
77
|
-
views, so the two paths are equivalent by construction.
|
|
78
|
-
"""
|
|
79
|
-
if self._read_from_v2:
|
|
80
|
-
return ("kgv2_nodes", "kgv2_edges")
|
|
81
|
-
return ("nodes", "edges")
|
|
82
|
-
|
|
83
|
-
@contextmanager
|
|
84
|
-
def _connect(self) -> Iterator[sqlite3.Connection]:
|
|
85
|
-
"""Transactional connection that is closed when the block exits.
|
|
86
|
-
|
|
87
|
-
This used to return the raw connection, so ``with self._connect() as
|
|
88
|
-
conn`` inherited sqlite3's context-manager semantics: commit on success,
|
|
89
|
-
rollback on error, **never close**. Each of the ~58 call sites therefore
|
|
90
|
-
held a descriptor open until garbage collection reclaimed it. That is
|
|
91
|
-
invisible while CPython refcounting frees the local immediately, and
|
|
92
|
-
fatal as soon as anything keeps the frame alive — a coverage tracer, a
|
|
93
|
-
profiler, or a traceback captured by ``logger.exception``.
|
|
94
|
-
|
|
95
|
-
Use ``storage_engine.connect()`` directly only for statements that
|
|
96
|
-
cannot run inside a transaction, and close it yourself.
|
|
97
|
-
"""
|
|
98
|
-
with self.storage_engine.session() as conn:
|
|
99
|
-
yield conn
|
|
100
|
-
|
|
101
|
-
def _init_db(self) -> None:
|
|
102
|
-
with self._connect() as conn:
|
|
103
|
-
db_format = int(conn.execute("PRAGMA user_version").fetchone()[0] or 0)
|
|
104
|
-
if db_format > _KG_DB_FORMAT_VERSION:
|
|
105
|
-
raise RuntimeError(
|
|
106
|
-
f"Knowledge Graph DB format {db_format} is newer than this build "
|
|
107
|
-
f"({_KG_DB_FORMAT_VERSION}); restore a pre-upgrade backup or upgrade Lattice AI."
|
|
108
|
-
)
|
|
109
|
-
conn.executescript(
|
|
110
|
-
"""
|
|
111
|
-
CREATE TABLE IF NOT EXISTS graph_meta (
|
|
112
|
-
key TEXT PRIMARY KEY,
|
|
113
|
-
value TEXT NOT NULL
|
|
114
|
-
);
|
|
115
|
-
CREATE TABLE IF NOT EXISTS nodes (
|
|
116
|
-
id TEXT PRIMARY KEY,
|
|
117
|
-
type TEXT NOT NULL,
|
|
118
|
-
title TEXT NOT NULL,
|
|
119
|
-
summary TEXT,
|
|
120
|
-
metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
|
|
121
|
-
raw_json TEXT NOT NULL CHECK (json_valid(raw_json)),
|
|
122
|
-
created_at TEXT NOT NULL,
|
|
123
|
-
updated_at TEXT NOT NULL
|
|
124
|
-
);
|
|
125
|
-
CREATE TABLE IF NOT EXISTS edges (
|
|
126
|
-
id TEXT PRIMARY KEY,
|
|
127
|
-
from_node TEXT NOT NULL,
|
|
128
|
-
to_node TEXT NOT NULL,
|
|
129
|
-
type TEXT NOT NULL,
|
|
130
|
-
weight REAL NOT NULL DEFAULT 1.0,
|
|
131
|
-
metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
|
|
132
|
-
created_at TEXT NOT NULL,
|
|
133
|
-
UNIQUE(from_node, to_node, type),
|
|
134
|
-
FOREIGN KEY(from_node) REFERENCES nodes(id) ON DELETE CASCADE,
|
|
135
|
-
FOREIGN KEY(to_node) REFERENCES nodes(id) ON DELETE CASCADE
|
|
136
|
-
);
|
|
137
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
138
|
-
id TEXT PRIMARY KEY,
|
|
139
|
-
source_node TEXT NOT NULL,
|
|
140
|
-
text TEXT NOT NULL,
|
|
141
|
-
metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
|
|
142
|
-
created_at TEXT NOT NULL,
|
|
143
|
-
FOREIGN KEY(source_node) REFERENCES nodes(id) ON DELETE CASCADE
|
|
144
|
-
);
|
|
145
|
-
CREATE TABLE IF NOT EXISTS knowledge_sources (
|
|
146
|
-
id TEXT PRIMARY KEY,
|
|
147
|
-
root_path TEXT NOT NULL UNIQUE,
|
|
148
|
-
os_type TEXT NOT NULL,
|
|
149
|
-
drive_id TEXT,
|
|
150
|
-
label TEXT,
|
|
151
|
-
status TEXT NOT NULL,
|
|
152
|
-
include_ocr INTEGER NOT NULL DEFAULT 0,
|
|
153
|
-
watch_enabled INTEGER NOT NULL DEFAULT 0,
|
|
154
|
-
consent_json TEXT NOT NULL CHECK (json_valid(consent_json)),
|
|
155
|
-
created_at TEXT NOT NULL,
|
|
156
|
-
updated_at TEXT NOT NULL,
|
|
157
|
-
last_scanned_at TEXT
|
|
158
|
-
);
|
|
159
|
-
CREATE TABLE IF NOT EXISTS local_file_index (
|
|
160
|
-
id TEXT PRIMARY KEY,
|
|
161
|
-
source_id TEXT NOT NULL,
|
|
162
|
-
os_type TEXT NOT NULL,
|
|
163
|
-
drive_id TEXT,
|
|
164
|
-
root_path TEXT NOT NULL,
|
|
165
|
-
file_path TEXT NOT NULL,
|
|
166
|
-
relative_path TEXT NOT NULL,
|
|
167
|
-
file_name TEXT NOT NULL,
|
|
168
|
-
extension TEXT NOT NULL,
|
|
169
|
-
size_bytes INTEGER,
|
|
170
|
-
modified_at TEXT,
|
|
171
|
-
sha256 TEXT,
|
|
172
|
-
last_scanned_at TEXT,
|
|
173
|
-
last_indexed_at TEXT,
|
|
174
|
-
parser_type TEXT,
|
|
175
|
-
status TEXT NOT NULL,
|
|
176
|
-
error_message TEXT,
|
|
177
|
-
graph_node_id TEXT,
|
|
178
|
-
deleted INTEGER NOT NULL DEFAULT 0,
|
|
179
|
-
metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
|
|
180
|
-
UNIQUE(source_id, relative_path),
|
|
181
|
-
FOREIGN KEY(source_id) REFERENCES knowledge_sources(id) ON DELETE CASCADE
|
|
182
|
-
);
|
|
183
|
-
CREATE TABLE IF NOT EXISTS vector_embeddings (
|
|
184
|
-
item_id TEXT PRIMARY KEY,
|
|
185
|
-
item_type TEXT NOT NULL,
|
|
186
|
-
source_node TEXT NOT NULL,
|
|
187
|
-
text_hash TEXT NOT NULL,
|
|
188
|
-
embedding BLOB NOT NULL,
|
|
189
|
-
embedding_dim INTEGER NOT NULL,
|
|
190
|
-
embedding_model TEXT NOT NULL,
|
|
191
|
-
metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
|
|
192
|
-
indexed_at TEXT NOT NULL,
|
|
193
|
-
FOREIGN KEY(source_node) REFERENCES nodes(id) ON DELETE CASCADE
|
|
194
|
-
);
|
|
195
|
-
CREATE TABLE IF NOT EXISTS vector_index_operations (
|
|
196
|
-
id TEXT PRIMARY KEY,
|
|
197
|
-
operation TEXT NOT NULL,
|
|
198
|
-
status TEXT NOT NULL,
|
|
199
|
-
requested_at TEXT NOT NULL,
|
|
200
|
-
started_at TEXT,
|
|
201
|
-
completed_at TEXT,
|
|
202
|
-
items_total INTEGER NOT NULL DEFAULT 0,
|
|
203
|
-
items_indexed INTEGER NOT NULL DEFAULT 0,
|
|
204
|
-
items_skipped INTEGER NOT NULL DEFAULT 0,
|
|
205
|
-
error_message TEXT,
|
|
206
|
-
metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json))
|
|
207
|
-
);
|
|
208
|
-
-- v3.6.0 Knowledge Graph First: per-ingestion provenance trail.
|
|
209
|
-
-- Append-only audit of where every graph node came from, when it
|
|
210
|
-
-- was captured, how it was processed, and whether it was embedded /
|
|
211
|
-
-- linked / used by an agent. get_provenance() returns the latest row.
|
|
212
|
-
CREATE TABLE IF NOT EXISTS ingestion_provenance (
|
|
213
|
-
id TEXT PRIMARY KEY,
|
|
214
|
-
node_id TEXT NOT NULL,
|
|
215
|
-
source_type TEXT NOT NULL,
|
|
216
|
-
source_uri TEXT,
|
|
217
|
-
content_hash TEXT,
|
|
218
|
-
title TEXT,
|
|
219
|
-
pipeline TEXT NOT NULL,
|
|
220
|
-
owner TEXT,
|
|
221
|
-
workspace_id TEXT,
|
|
222
|
-
captured_at TEXT,
|
|
223
|
-
modified_at TEXT,
|
|
224
|
-
embedded INTEGER NOT NULL DEFAULT 0,
|
|
225
|
-
linked INTEGER NOT NULL DEFAULT 0,
|
|
226
|
-
duplicate INTEGER NOT NULL DEFAULT 0,
|
|
227
|
-
agent_used TEXT,
|
|
228
|
-
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
229
|
-
permissions_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(permissions_json)),
|
|
230
|
-
metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)),
|
|
231
|
-
created_at TEXT NOT NULL
|
|
232
|
-
);
|
|
233
|
-
CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
|
|
234
|
-
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node);
|
|
235
|
-
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node);
|
|
236
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_node);
|
|
237
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_sources_root ON knowledge_sources(root_path);
|
|
238
|
-
CREATE INDEX IF NOT EXISTS idx_local_file_index_source ON local_file_index(source_id);
|
|
239
|
-
CREATE INDEX IF NOT EXISTS idx_local_file_index_status ON local_file_index(status);
|
|
240
|
-
CREATE INDEX IF NOT EXISTS idx_local_file_index_graph_node ON local_file_index(graph_node_id);
|
|
241
|
-
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_type ON vector_embeddings(item_type);
|
|
242
|
-
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_source ON vector_embeddings(source_node);
|
|
243
|
-
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_model ON vector_embeddings(embedding_model);
|
|
244
|
-
-- v11.2.0: the ANN sidecar's freshness fingerprint is
|
|
245
|
-
-- COUNT(*) + MAX(indexed_at) filtered by (embedding_model,
|
|
246
|
-
-- embedding_dim), and it runs on *every* approximate
|
|
247
|
-
-- search. Against the single-column model index that was a
|
|
248
|
-
-- table scan of every row for the current model; this
|
|
249
|
-
-- covering index answers both aggregates from the index
|
|
250
|
-
-- alone. Additive and idempotent — no data is rewritten.
|
|
251
|
-
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_model_dim_indexed
|
|
252
|
-
ON vector_embeddings(embedding_model, embedding_dim, indexed_at);
|
|
253
|
-
CREATE INDEX IF NOT EXISTS idx_vector_index_operations_requested ON vector_index_operations(requested_at);
|
|
254
|
-
CREATE INDEX IF NOT EXISTS idx_provenance_node ON ingestion_provenance(node_id);
|
|
255
|
-
CREATE INDEX IF NOT EXISTS idx_provenance_source_type ON ingestion_provenance(source_type);
|
|
256
|
-
CREATE INDEX IF NOT EXISTS idx_provenance_hash ON ingestion_provenance(content_hash);
|
|
257
|
-
CREATE INDEX IF NOT EXISTS idx_provenance_created ON ingestion_provenance(created_at);
|
|
258
|
-
"""
|
|
259
|
-
)
|
|
260
|
-
conn.execute(
|
|
261
|
-
"INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
|
|
262
|
-
("schema_version", str(GRAPH_SCHEMA_VERSION)),
|
|
263
|
-
)
|
|
264
|
-
self._init_v2_schema()
|
|
265
|
-
self._init_fts()
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
__all__ = ["KnowledgeGraphStore"]
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
"""Pluggable vector-index backends + the durable pending-embed queue.
|
|
2
|
-
|
|
3
|
-
Public surface (v11.1.0, Track 1):
|
|
4
|
-
|
|
5
|
-
* :class:`VectorIndex` — the Protocol every backend satisfies.
|
|
6
|
-
* :class:`BruteForceIndex` — exact, exhaustive, the default.
|
|
7
|
-
* :class:`QuantizedIndex` — int8 storage, exhaustive, approximate scores.
|
|
8
|
-
* :class:`HnswIndex` — approximate nearest neighbour, optional ``hnswlib``.
|
|
9
|
-
* :func:`resolve_vector_index` / :func:`build_index` — ``LATTICEAI_VECTOR_INDEX``
|
|
10
|
-
resolution with an honest fallback reason.
|
|
11
|
-
* :class:`VectorEmbedQueue` — SQLite-backed background embedding backlog.
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
from __future__ import annotations
|
|
15
|
-
|
|
16
|
-
from .base import (
|
|
17
|
-
IndexItem,
|
|
18
|
-
IndexStats,
|
|
19
|
-
ScoredId,
|
|
20
|
-
Similarity,
|
|
21
|
-
VectorIndex,
|
|
22
|
-
dot_similarity,
|
|
23
|
-
score_floor,
|
|
24
|
-
take_top,
|
|
25
|
-
)
|
|
26
|
-
from .brute_force import BRUTE_FORCE_BACKEND, BruteForceIndex
|
|
27
|
-
from .hnsw import (
|
|
28
|
-
HNSW_BACKEND,
|
|
29
|
-
HNSW_META_SUFFIX,
|
|
30
|
-
HNSW_SUFFIX,
|
|
31
|
-
HnswIndex,
|
|
32
|
-
hnswlib_available,
|
|
33
|
-
load_hnswlib,
|
|
34
|
-
sidecar_paths,
|
|
35
|
-
)
|
|
36
|
-
from .jobs import (
|
|
37
|
-
DEFAULT_MAX_ATTEMPTS,
|
|
38
|
-
DEFAULT_TICK_LIMIT,
|
|
39
|
-
VECTOR_JOB_STATUSES,
|
|
40
|
-
VectorEmbedQueue,
|
|
41
|
-
VectorJobStore,
|
|
42
|
-
)
|
|
43
|
-
from .quantized import QUANTIZED_BACKEND, QuantizedIndex, quantize
|
|
44
|
-
from .selector import (
|
|
45
|
-
DEFAULT_VECTOR_INDEX,
|
|
46
|
-
VECTOR_INDEX_CHOICES,
|
|
47
|
-
VECTOR_INDEX_ENV,
|
|
48
|
-
BackendSelection,
|
|
49
|
-
build_index,
|
|
50
|
-
resolve_vector_index,
|
|
51
|
-
)
|
|
52
|
-
|
|
53
|
-
__all__ = [
|
|
54
|
-
"BRUTE_FORCE_BACKEND",
|
|
55
|
-
"DEFAULT_MAX_ATTEMPTS",
|
|
56
|
-
"DEFAULT_TICK_LIMIT",
|
|
57
|
-
"DEFAULT_VECTOR_INDEX",
|
|
58
|
-
"HNSW_BACKEND",
|
|
59
|
-
"HNSW_META_SUFFIX",
|
|
60
|
-
"HNSW_SUFFIX",
|
|
61
|
-
"QUANTIZED_BACKEND",
|
|
62
|
-
"VECTOR_INDEX_CHOICES",
|
|
63
|
-
"VECTOR_INDEX_ENV",
|
|
64
|
-
"VECTOR_JOB_STATUSES",
|
|
65
|
-
"BackendSelection",
|
|
66
|
-
"BruteForceIndex",
|
|
67
|
-
"HnswIndex",
|
|
68
|
-
"IndexItem",
|
|
69
|
-
"IndexStats",
|
|
70
|
-
"QuantizedIndex",
|
|
71
|
-
"ScoredId",
|
|
72
|
-
"Similarity",
|
|
73
|
-
"VectorEmbedQueue",
|
|
74
|
-
"VectorIndex",
|
|
75
|
-
"VectorJobStore",
|
|
76
|
-
"build_index",
|
|
77
|
-
"dot_similarity",
|
|
78
|
-
"hnswlib_available",
|
|
79
|
-
"load_hnswlib",
|
|
80
|
-
"quantize",
|
|
81
|
-
"resolve_vector_index",
|
|
82
|
-
"score_floor",
|
|
83
|
-
"sidecar_paths",
|
|
84
|
-
"take_top",
|
|
85
|
-
]
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
"""The vector-index contract: what it means to score embeddings (v11.1.0).
|
|
2
|
-
|
|
3
|
-
Before this package the only vector "index" was the scan loop inside
|
|
4
|
-
``retrieval_vector.vector_search``: fetch every embedding row, decode it,
|
|
5
|
-
score it, sort. That is a perfectly good *exact* index and it stays the
|
|
6
|
-
default — but it was welded to SQL row handling, so there was nowhere to put
|
|
7
|
-
a second strategy without forking the search.
|
|
8
|
-
|
|
9
|
-
This module defines the seam. A backend owns exactly one question — *given a
|
|
10
|
-
query vector, which stored ids score highest* — and nothing about SQLite,
|
|
11
|
-
chunk metadata, workspaces, or citations. The vectors are ``Sequence[float]``
|
|
12
|
-
(the repo's embedders return ``list[float]``; numpy is not a dependency).
|
|
13
|
-
|
|
14
|
-
Honesty contract: every backend reports whether it is ``approx`` (scores are
|
|
15
|
-
estimates) and whether it is ``exhaustive`` (every stored vector was
|
|
16
|
-
compared). Those two bits travel with the search result so a caller never has
|
|
17
|
-
to guess whether "no match" means "not in your brain" or "the index did not
|
|
18
|
-
look".
|
|
19
|
-
"""
|
|
20
|
-
|
|
21
|
-
from __future__ import annotations
|
|
22
|
-
|
|
23
|
-
from dataclasses import dataclass
|
|
24
|
-
from typing import (
|
|
25
|
-
Any,
|
|
26
|
-
Callable,
|
|
27
|
-
Dict,
|
|
28
|
-
Iterable,
|
|
29
|
-
List,
|
|
30
|
-
Mapping,
|
|
31
|
-
Optional,
|
|
32
|
-
Protocol,
|
|
33
|
-
Sequence,
|
|
34
|
-
Tuple,
|
|
35
|
-
)
|
|
36
|
-
|
|
37
|
-
#: ``(item_id, score)`` — the one shape every backend returns.
|
|
38
|
-
ScoredId = Tuple[str, float]
|
|
39
|
-
#: ``(item_id, vector, metadata)`` — one item as handed to :meth:`rebuild`.
|
|
40
|
-
IndexItem = Tuple[str, Sequence[float], Mapping[str, Any]]
|
|
41
|
-
#: Pluggable similarity so a backend can borrow the embedder's own function.
|
|
42
|
-
Similarity = Callable[[Sequence[float], Sequence[float]], float]
|
|
43
|
-
|
|
44
|
-
#: "no score floor" — ``0.0`` is a meaningful floor, so it cannot be the
|
|
45
|
-
#: sentinel (the score-0-is-falsy trap this codebase keeps re-learning).
|
|
46
|
-
NO_FLOOR = float("-inf")
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def dot_similarity(left: Sequence[float], right: Sequence[float]) -> float:
|
|
50
|
-
"""Dot product, which is cosine similarity for unit-length vectors.
|
|
51
|
-
|
|
52
|
-
Mirrors :meth:`lattice_brain.embeddings.LocalEmbeddingModel.similarity`,
|
|
53
|
-
including its refusal to compare different dimensions: truncating to the
|
|
54
|
-
shorter vector produces a plausible-looking number that means nothing.
|
|
55
|
-
"""
|
|
56
|
-
left_v, right_v = list(left), list(right)
|
|
57
|
-
if len(left_v) != len(right_v):
|
|
58
|
-
raise ValueError(
|
|
59
|
-
f"embedding dimension mismatch: {len(left_v)} vs {len(right_v)}; "
|
|
60
|
-
"the vector index was built with a different model"
|
|
61
|
-
)
|
|
62
|
-
return float(sum(a * b for a, b in zip(left_v, right_v, strict=True)))
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def score_floor(filter: Optional[Mapping[str, Any]]) -> float:
|
|
66
|
-
"""Resolve ``filter["min_score"]`` (absent/unparseable → no floor)."""
|
|
67
|
-
if not filter:
|
|
68
|
-
return NO_FLOOR
|
|
69
|
-
raw = filter.get("min_score")
|
|
70
|
-
if raw is None:
|
|
71
|
-
return NO_FLOOR
|
|
72
|
-
try:
|
|
73
|
-
return float(raw)
|
|
74
|
-
except (TypeError, ValueError):
|
|
75
|
-
return NO_FLOOR
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
def take_top(scored: List[ScoredId], top_k: int) -> List[ScoredId]:
|
|
79
|
-
"""Sort by score descending (stable → insertion order breaks ties), cut."""
|
|
80
|
-
scored.sort(key=lambda pair: -pair[1])
|
|
81
|
-
return scored[: max(0, int(top_k))]
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
@dataclass(frozen=True)
|
|
85
|
-
class IndexStats:
|
|
86
|
-
"""What a backend is willing to claim about itself."""
|
|
87
|
-
|
|
88
|
-
backend: str
|
|
89
|
-
size: int
|
|
90
|
-
dim: int
|
|
91
|
-
approx: bool
|
|
92
|
-
exhaustive: bool
|
|
93
|
-
available: bool = True
|
|
94
|
-
detail: Optional[str] = None
|
|
95
|
-
|
|
96
|
-
def as_dict(self) -> Dict[str, Any]:
|
|
97
|
-
return {
|
|
98
|
-
"backend": self.backend,
|
|
99
|
-
"size": self.size,
|
|
100
|
-
"dim": self.dim,
|
|
101
|
-
"approx": self.approx,
|
|
102
|
-
"exhaustive": self.exhaustive,
|
|
103
|
-
"available": self.available,
|
|
104
|
-
"detail": self.detail,
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
class VectorIndex(Protocol):
|
|
109
|
-
"""Structural contract implemented by every backend in this package.
|
|
110
|
-
|
|
111
|
-
Declared, never implemented — same convention as
|
|
112
|
-
:mod:`lattice_brain.graph._kg_contract`: the bodies raise so a stub can
|
|
113
|
-
never be mistaken for a working index.
|
|
114
|
-
"""
|
|
115
|
-
|
|
116
|
-
@property
|
|
117
|
-
def backend(self) -> str:
|
|
118
|
-
"""Stable id of the scoring strategy (goes into ``recall.backend``)."""
|
|
119
|
-
raise NotImplementedError
|
|
120
|
-
|
|
121
|
-
@property
|
|
122
|
-
def approx(self) -> bool:
|
|
123
|
-
"""True when scores are estimates rather than exact similarities."""
|
|
124
|
-
raise NotImplementedError
|
|
125
|
-
|
|
126
|
-
@property
|
|
127
|
-
def exhaustive(self) -> bool:
|
|
128
|
-
"""True when every stored vector is compared on every search."""
|
|
129
|
-
raise NotImplementedError
|
|
130
|
-
|
|
131
|
-
def add(
|
|
132
|
-
self,
|
|
133
|
-
id: str,
|
|
134
|
-
vector: Sequence[float],
|
|
135
|
-
metadata: Optional[Mapping[str, Any]] = None,
|
|
136
|
-
) -> None:
|
|
137
|
-
raise NotImplementedError
|
|
138
|
-
|
|
139
|
-
def remove(self, id: str) -> None:
|
|
140
|
-
raise NotImplementedError
|
|
141
|
-
|
|
142
|
-
def rebuild(self, items: Iterable[IndexItem]) -> None:
|
|
143
|
-
raise NotImplementedError
|
|
144
|
-
|
|
145
|
-
def search(
|
|
146
|
-
self,
|
|
147
|
-
query: Sequence[float],
|
|
148
|
-
top_k: int,
|
|
149
|
-
filter: Optional[Mapping[str, Any]] = None,
|
|
150
|
-
) -> List[ScoredId]:
|
|
151
|
-
raise NotImplementedError
|
|
152
|
-
|
|
153
|
-
def stats(self) -> IndexStats:
|
|
154
|
-
raise NotImplementedError
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
__all__ = [
|
|
158
|
-
"NO_FLOOR",
|
|
159
|
-
"IndexItem",
|
|
160
|
-
"IndexStats",
|
|
161
|
-
"ScoredId",
|
|
162
|
-
"Similarity",
|
|
163
|
-
"VectorIndex",
|
|
164
|
-
"dot_similarity",
|
|
165
|
-
"score_floor",
|
|
166
|
-
"take_top",
|
|
167
|
-
]
|
|
@@ -1,110 +0,0 @@
|
|
|
1
|
-
"""Exact cosine scan — the historical ``vector_search`` scoring loop, moved.
|
|
2
|
-
|
|
3
|
-
This backend is deliberately boring and deliberately the default. It compares
|
|
4
|
-
the query against every vector it holds, so its answer is the ground truth the
|
|
5
|
-
other backends are measured against (``scripts/bench_vector_index.py`` reports
|
|
6
|
-
recall@k relative to this one).
|
|
7
|
-
|
|
8
|
-
Behaviour parity with the pre-11.1.0 inline loop is the whole point of the
|
|
9
|
-
move: same similarity function (injected, so it is the embedder's own), same
|
|
10
|
-
``score < min_score`` drop, same stable ordering. ``retrieval_vector`` feeds
|
|
11
|
-
it in bounded batches so a scan never holds every decoded vector at once.
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
from __future__ import annotations
|
|
15
|
-
|
|
16
|
-
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence
|
|
17
|
-
|
|
18
|
-
from .base import (
|
|
19
|
-
IndexItem,
|
|
20
|
-
IndexStats,
|
|
21
|
-
ScoredId,
|
|
22
|
-
Similarity,
|
|
23
|
-
dot_similarity,
|
|
24
|
-
score_floor,
|
|
25
|
-
take_top,
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
BRUTE_FORCE_BACKEND = "bruteforce-cosine"
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class BruteForceIndex:
|
|
32
|
-
"""Exhaustive, exact, in-memory cosine index."""
|
|
33
|
-
|
|
34
|
-
def __init__(
|
|
35
|
-
self,
|
|
36
|
-
*,
|
|
37
|
-
dim: int = 0,
|
|
38
|
-
similarity: Optional[Similarity] = None,
|
|
39
|
-
) -> None:
|
|
40
|
-
self._dim = int(dim)
|
|
41
|
-
self._similarity: Similarity = (
|
|
42
|
-
dot_similarity if similarity is None else similarity
|
|
43
|
-
)
|
|
44
|
-
self._vectors: Dict[str, List[float]] = {}
|
|
45
|
-
self._metadata: Dict[str, Dict[str, Any]] = {}
|
|
46
|
-
|
|
47
|
-
@property
|
|
48
|
-
def backend(self) -> str:
|
|
49
|
-
return BRUTE_FORCE_BACKEND
|
|
50
|
-
|
|
51
|
-
@property
|
|
52
|
-
def approx(self) -> bool:
|
|
53
|
-
return False
|
|
54
|
-
|
|
55
|
-
@property
|
|
56
|
-
def exhaustive(self) -> bool:
|
|
57
|
-
return True
|
|
58
|
-
|
|
59
|
-
def add(
|
|
60
|
-
self,
|
|
61
|
-
id: str,
|
|
62
|
-
vector: Sequence[float],
|
|
63
|
-
metadata: Optional[Mapping[str, Any]] = None,
|
|
64
|
-
) -> None:
|
|
65
|
-
values = [float(value) for value in vector]
|
|
66
|
-
if not self._dim:
|
|
67
|
-
self._dim = len(values)
|
|
68
|
-
key = str(id)
|
|
69
|
-
self._vectors[key] = values
|
|
70
|
-
self._metadata[key] = dict(metadata or {})
|
|
71
|
-
|
|
72
|
-
def remove(self, id: str) -> None:
|
|
73
|
-
key = str(id)
|
|
74
|
-
self._vectors.pop(key, None)
|
|
75
|
-
self._metadata.pop(key, None)
|
|
76
|
-
|
|
77
|
-
def rebuild(self, items: Iterable[IndexItem]) -> None:
|
|
78
|
-
self._vectors.clear()
|
|
79
|
-
self._metadata.clear()
|
|
80
|
-
for item_id, vector, metadata in items:
|
|
81
|
-
self.add(item_id, vector, metadata)
|
|
82
|
-
|
|
83
|
-
def search(
|
|
84
|
-
self,
|
|
85
|
-
query: Sequence[float],
|
|
86
|
-
top_k: int,
|
|
87
|
-
filter: Optional[Mapping[str, Any]] = None,
|
|
88
|
-
) -> List[ScoredId]:
|
|
89
|
-
floor = score_floor(filter)
|
|
90
|
-
query_vector = list(query)
|
|
91
|
-
scored: List[ScoredId] = []
|
|
92
|
-
for item_id, vector in self._vectors.items():
|
|
93
|
-
score = float(self._similarity(query_vector, vector))
|
|
94
|
-
if score < floor:
|
|
95
|
-
continue
|
|
96
|
-
scored.append((item_id, score))
|
|
97
|
-
return take_top(scored, top_k)
|
|
98
|
-
|
|
99
|
-
def stats(self) -> IndexStats:
|
|
100
|
-
return IndexStats(
|
|
101
|
-
backend=self.backend,
|
|
102
|
-
size=len(self._vectors),
|
|
103
|
-
dim=self._dim,
|
|
104
|
-
approx=False,
|
|
105
|
-
exhaustive=True,
|
|
106
|
-
detail=None,
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
__all__ = ["BRUTE_FORCE_BACKEND", "BruteForceIndex"]
|