@lapis-notes/api 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/LICENSE.md +4 -0
- package/README.md +43 -0
- package/dist/__tests__/data-adapter-conformance.js +205 -0
- package/dist/__tests__/lapis-runtime-manifest.js +32 -0
- package/dist/__tests__/svelte-internal-client.d.ts +3 -0
- package/dist/agent-result-views.d.ts +52 -0
- package/dist/agent-result-views.js +89 -0
- package/dist/agent-skills.d.ts +99 -0
- package/dist/agent-skills.js +182 -0
- package/dist/agent-tools.d.ts +92 -0
- package/dist/agent-tools.js +139 -0
- package/dist/app-database-host.d.ts +4 -0
- package/dist/app-database-host.js +4 -0
- package/dist/app-shell-plugin-persistence.d.ts +15 -0
- package/dist/app-shell-plugin-persistence.js +34 -0
- package/dist/app-url.d.ts +54 -0
- package/dist/app-url.js +269 -0
- package/dist/application-compatibility.d.ts +31 -0
- package/dist/application-compatibility.js +64 -0
- package/dist/application-state.svelte.d.ts +14 -0
- package/dist/application-state.svelte.js +30 -0
- package/dist/assets/lapis-logo.svg +471 -0
- package/dist/bases.d.ts +281 -0
- package/dist/bases.js +475 -0
- package/dist/cache.svelte.d.ts +569 -0
- package/dist/cache.svelte.js +1433 -0
- package/dist/command.svelte.d.ts +337 -0
- package/dist/command.svelte.js +571 -0
- package/dist/compat.d.ts +29 -0
- package/dist/compat.js +30 -0
- package/dist/components/configuration/configuration.css +199 -0
- package/dist/components/configuration/date-setting.svelte +71 -0
- package/dist/components/configuration/date-setting.svelte.d.ts +8 -0
- package/dist/components/configuration/list-setting.svelte +128 -0
- package/dist/components/configuration/list-setting.svelte.d.ts +13 -0
- package/dist/components/configuration/object-array-row-grip.svelte +78 -0
- package/dist/components/configuration/object-array-row-grip.svelte.d.ts +9 -0
- package/dist/components/configuration/object-array-row.svelte +163 -0
- package/dist/components/configuration/object-array-row.svelte.d.ts +19 -0
- package/dist/components/configuration/object-array-setting.svelte +196 -0
- package/dist/components/configuration/object-array-setting.svelte.d.ts +15 -0
- package/dist/components/configuration/object-array-table-dnd.d.ts +8 -0
- package/dist/components/configuration/object-array-table-dnd.js +10 -0
- package/dist/components/configuration/object-array-utils.d.ts +11 -0
- package/dist/components/configuration/object-array-utils.js +103 -0
- package/dist/components/configuration/object-grid-setting.svelte +79 -0
- package/dist/components/configuration/object-grid-setting.svelte.d.ts +10 -0
- package/dist/components/configuration/object-map-row.svelte +147 -0
- package/dist/components/configuration/object-map-row.svelte.d.ts +15 -0
- package/dist/components/configuration/object-map-setting.svelte +137 -0
- package/dist/components/configuration/object-map-setting.svelte.d.ts +14 -0
- package/dist/components/configuration/object-map-types.d.ts +5 -0
- package/dist/components/configuration/object-map-types.js +1 -0
- package/dist/components/configuration/options-combobox-setting.svelte +129 -0
- package/dist/components/configuration/options-combobox-setting.svelte.d.ts +12 -0
- package/dist/components/copy-button/button.svelte +93 -0
- package/dist/components/copy-button/button.svelte.d.ts +13 -0
- package/dist/components/copy-button/index.d.ts +2 -0
- package/dist/components/copy-button/index.js +2 -0
- package/dist/components/editor/editor.css +565 -0
- package/dist/components/editor/editor.d.ts +33 -0
- package/dist/components/editor/editor.js +132 -0
- package/dist/components/editor/editor.svelte +176 -0
- package/dist/components/editor/editor.svelte.d.ts +15 -0
- package/dist/components/editor/embedded-editor-surface.d.ts +12 -0
- package/dist/components/editor/embedded-editor-surface.js +22 -0
- package/dist/components/editor/embedded-editor-surface.svelte +115 -0
- package/dist/components/editor/embedded-editor-surface.svelte.d.ts +17 -0
- package/dist/components/editor/extensions/autocomplete/index.d.ts +2 -0
- package/dist/components/editor/extensions/autocomplete/index.js +2 -0
- package/dist/components/editor/extensions/autocomplete/lapis-code-mirror-autocomplete.d.ts +16 -0
- package/dist/components/editor/extensions/autocomplete/lapis-code-mirror-autocomplete.js +34 -0
- package/dist/components/editor/extensions/autocomplete/vault-completion-sources.d.ts +27 -0
- package/dist/components/editor/extensions/autocomplete/vault-completion-sources.js +55 -0
- package/dist/components/editor/extensions/class-highlighter/index.d.ts +1 -0
- package/dist/components/editor/extensions/class-highlighter/index.js +47 -0
- package/dist/components/editor/extensions/fold-gutter/index.d.ts +8 -0
- package/dist/components/editor/extensions/fold-gutter/index.js +40 -0
- package/dist/components/editor/extensions/gutter-line-style-sync/index.d.ts +2 -0
- package/dist/components/editor/extensions/gutter-line-style-sync/index.js +118 -0
- package/dist/components/editor/extensions/lint/index.d.ts +4 -0
- package/dist/components/editor/extensions/lint/index.js +4 -0
- package/dist/components/editor/extensions/lint/lapis-code-mirror-lint.d.ts +26 -0
- package/dist/components/editor/extensions/lint/lapis-code-mirror-lint.js +101 -0
- package/dist/components/editor/extensions/lint/lapis-lint-diagnostic-helpers.d.ts +8 -0
- package/dist/components/editor/extensions/lint/lapis-lint-diagnostic-helpers.js +7 -0
- package/dist/components/editor/extensions/lint/lapis-lint-diagnostic.d.ts +20 -0
- package/dist/components/editor/extensions/lint/lapis-lint-diagnostic.js +47 -0
- package/dist/components/editor/extensions/lint/lapis-lint-hover-tooltip.d.ts +9 -0
- package/dist/components/editor/extensions/lint/lapis-lint-hover-tooltip.js +373 -0
- package/dist/components/editor/extensions/lint/lapis-lint-inline-widget.d.ts +31 -0
- package/dist/components/editor/extensions/lint/lapis-lint-inline-widget.js +191 -0
- package/dist/components/editor/extensions/lint/lapis-lint-tooltip.svelte +176 -0
- package/dist/components/editor/extensions/lint/lapis-lint-tooltip.svelte.d.ts +15 -0
- package/dist/components/editor/extensions/lint/lint-tooltip-actions.d.ts +11 -0
- package/dist/components/editor/extensions/lint/lint-tooltip-actions.js +21 -0
- package/dist/components/editor/extensions/lint/mount-lint-tooltip.d.ts +16 -0
- package/dist/components/editor/extensions/lint/mount-lint-tooltip.js +60 -0
- package/dist/components/editor/extensions/search/dom.d.ts +2 -0
- package/dist/components/editor/extensions/search/dom.js +6 -0
- package/dist/components/editor/extensions/search/index.d.ts +35 -0
- package/dist/components/editor/extensions/search/index.js +44 -0
- package/dist/components/editor/extensions/search/search.svelte +335 -0
- package/dist/components/editor/extensions/search/search.svelte.d.ts +8 -0
- package/dist/components/editor/index.d.ts +9 -0
- package/dist/components/editor/index.js +9 -0
- package/dist/components/editor/language-service/dom.d.ts +1 -0
- package/dist/components/editor/language-service/dom.js +6 -0
- package/dist/components/editor/language-service/index.d.ts +28 -0
- package/dist/components/editor/language-service/index.js +342 -0
- package/dist/components/empty-view/empty-view.css +39 -0
- package/dist/components/empty-view/empty-view.svelte +26 -0
- package/dist/components/empty-view/empty-view.svelte.d.ts +9 -0
- package/dist/components/empty-view/index.d.ts +2 -0
- package/dist/components/empty-view/index.js +2 -0
- package/dist/components/icon/icon-list.css +74 -0
- package/dist/components/icon/icon-list.svelte +213 -0
- package/dist/components/icon/icon-list.svelte.d.ts +11 -0
- package/dist/components/icon/icon.svelte +15 -0
- package/dist/components/icon/icon.svelte.d.ts +7 -0
- package/dist/components/icon/index.d.ts +3 -0
- package/dist/components/icon/index.js +3 -0
- package/dist/components/menu/context-menu.svelte +89 -0
- package/dist/components/menu/context-menu.svelte.d.ts +11 -0
- package/dist/components/menu/drawer-menu.svelte +83 -0
- package/dist/components/menu/drawer-menu.svelte.d.ts +8 -0
- package/dist/components/menu/dropdown-menu.svelte +89 -0
- package/dist/components/menu/dropdown-menu.svelte.d.ts +11 -0
- package/dist/components/menu/index.d.ts +6 -0
- package/dist/components/menu/index.js +6 -0
- package/dist/components/menu/menu.css +139 -0
- package/dist/components/menu/menu.svelte +37 -0
- package/dist/components/menu/menu.svelte.d.ts +11 -0
- package/dist/components/menu/mobile-menu-query.svelte.d.ts +3 -0
- package/dist/components/menu/mobile-menu-query.svelte.js +21 -0
- package/dist/components/menu/popover-menu.svelte +90 -0
- package/dist/components/menu/popover-menu.svelte.d.ts +10 -0
- package/dist/components/select/index.d.ts +2 -0
- package/dist/components/select/index.js +2 -0
- package/dist/components/select/select.svelte +72 -0
- package/dist/components/select/select.svelte.d.ts +25 -0
- package/dist/configuration-option-source-providers.d.ts +17 -0
- package/dist/configuration-option-source-providers.js +59 -0
- package/dist/configuration-option-source-registry.d.ts +59 -0
- package/dist/configuration-option-source-registry.js +123 -0
- package/dist/configuration.svelte.d.ts +258 -0
- package/dist/configuration.svelte.js +887 -0
- package/dist/context-keys.svelte.d.ts +42 -0
- package/dist/context-keys.svelte.js +278 -0
- package/dist/context.svelte.d.ts +247 -0
- package/dist/context.svelte.js +540 -0
- package/dist/daily-document-provider.d.ts +38 -0
- package/dist/daily-document-provider.js +176 -0
- package/dist/design-core-button-types.d.ts +7 -0
- package/dist/design-core-button-types.js +1 -0
- package/dist/diagnostics.d.ts +93 -0
- package/dist/diagnostics.js +84 -0
- package/dist/dialog-portal.d.ts +3 -0
- package/dist/dialog-portal.js +3 -0
- package/dist/docs-api.d.ts +44 -0
- package/dist/docs-api.js +29 -0
- package/dist/editor-view-registry.d.ts +35 -0
- package/dist/editor-view-registry.js +112 -0
- package/dist/editor.svelte.d.ts +126 -0
- package/dist/editor.svelte.js +454 -0
- package/dist/embed-registry.d.ts +15 -0
- package/dist/embed-registry.js +16 -0
- package/dist/enhance.d.ts +111 -0
- package/dist/enhance.js +309 -0
- package/dist/events.d.ts +28 -0
- package/dist/events.js +49 -0
- package/dist/file-manager.d.ts +90 -0
- package/dist/file-manager.js +289 -0
- package/dist/file-tools/app-tools.d.ts +3 -0
- package/dist/file-tools/app-tools.js +242 -0
- package/dist/file-tools/index.d.ts +3 -0
- package/dist/file-tools/index.js +3 -0
- package/dist/file-tools/paths.d.ts +14 -0
- package/dist/file-tools/paths.js +33 -0
- package/dist/file-tools/vault-operations.d.ts +3 -0
- package/dist/file-tools/vault-operations.js +60 -0
- package/dist/generated/plugin-host-modules.schema.generated.d.ts +371 -0
- package/dist/generated/plugin-host-modules.schema.generated.js +374 -0
- package/dist/glob.d.ts +15 -0
- package/dist/glob.js +68 -0
- package/dist/history.svelte.d.ts +24 -0
- package/dist/history.svelte.js +74 -0
- package/dist/hooks/createVirtualizer.svelte.d.ts +7 -0
- package/dist/hooks/createVirtualizer.svelte.js +70 -0
- package/dist/hooks/index.d.ts +5 -0
- package/dist/hooks/index.js +5 -0
- package/dist/hooks/is-mobile.svelte.d.ts +5 -0
- package/dist/hooks/is-mobile.svelte.js +23 -0
- package/dist/hooks/mountComponent.svelte.d.ts +7 -0
- package/dist/hooks/mountComponent.svelte.js +16 -0
- package/dist/hooks/use-text-highlight.d.ts +17 -0
- package/dist/hooks/use-text-highlight.js +82 -0
- package/dist/hooks/useBoundingRect.svelte.d.ts +11 -0
- package/dist/hooks/useBoundingRect.svelte.js +46 -0
- package/dist/icons/index.d.ts +57 -0
- package/dist/icons/index.js +288 -0
- package/dist/index.d.ts +87 -0
- package/dist/index.js +87 -0
- package/dist/language-service/index.d.ts +9 -0
- package/dist/language-service/index.js +7 -0
- package/dist/language-service/manager.d.ts +47 -0
- package/dist/language-service/manager.js +404 -0
- package/dist/language-service/types.d.ts +92 -0
- package/dist/language-service/types.js +1 -0
- package/dist/language-service/virtual-document.d.ts +11 -0
- package/dist/language-service/virtual-document.js +45 -0
- package/dist/language-service/worker-protocol.d.ts +63 -0
- package/dist/language-service/worker-protocol.js +1 -0
- package/dist/language-service/worker-provider.d.ts +9 -0
- package/dist/language-service/worker-provider.js +38 -0
- package/dist/lapis-extension.d.ts +244 -0
- package/dist/lapis-extension.js +591 -0
- package/dist/links.d.ts +34 -0
- package/dist/links.js +191 -0
- package/dist/loaders.d.ts +9 -0
- package/dist/loaders.js +59 -0
- package/dist/localization-manager.svelte.d.ts +56 -0
- package/dist/localization-manager.svelte.js +157 -0
- package/dist/logging.d.ts +172 -0
- package/dist/logging.js +352 -0
- package/dist/markdown-extension-registry.d.ts +31 -0
- package/dist/markdown-extension-registry.js +31 -0
- package/dist/markdown-file-surface-registry.d.ts +25 -0
- package/dist/markdown-file-surface-registry.js +23 -0
- package/dist/markdown.d.ts +205 -0
- package/dist/markdown.js +147 -0
- package/dist/menu-utils.d.ts +1 -0
- package/dist/menu-utils.js +47 -0
- package/dist/menu.svelte.d.ts +121 -0
- package/dist/menu.svelte.js +295 -0
- package/dist/metadata-value.d.ts +10 -0
- package/dist/metadata-value.js +173 -0
- package/dist/metadata.svelte.d.ts +106 -0
- package/dist/metadata.svelte.js +608 -0
- package/dist/notifications.d.ts +209 -0
- package/dist/notifications.js +338 -0
- package/dist/open-file.d.ts +23 -0
- package/dist/open-file.js +102 -0
- package/dist/platform.d.ts +86 -0
- package/dist/platform.js +86 -0
- package/dist/plugin-asset-server.d.ts +60 -0
- package/dist/plugin-asset-server.js +222 -0
- package/dist/plugin-assets.d.ts +4 -0
- package/dist/plugin-assets.js +3 -0
- package/dist/plugin-capability-facade.d.ts +104 -0
- package/dist/plugin-capability-facade.js +61 -0
- package/dist/plugin-dependency-resolver.d.ts +36 -0
- package/dist/plugin-dependency-resolver.js +54 -0
- package/dist/plugin-dependency-scanner.d.ts +9 -0
- package/dist/plugin-dependency-scanner.js +274 -0
- package/dist/plugin-distribution/canonical-json.d.ts +1 -0
- package/dist/plugin-distribution/canonical-json.js +40 -0
- package/dist/plugin-distribution/compatibility.d.ts +5 -0
- package/dist/plugin-distribution/compatibility.js +38 -0
- package/dist/plugin-distribution/errors.d.ts +11 -0
- package/dist/plugin-distribution/errors.js +13 -0
- package/dist/plugin-distribution/hashes.d.ts +2 -0
- package/dist/plugin-distribution/hashes.js +17 -0
- package/dist/plugin-distribution/index.d.ts +18 -0
- package/dist/plugin-distribution/index.js +18 -0
- package/dist/plugin-distribution/installed-plugin-state.d.ts +20 -0
- package/dist/plugin-distribution/installed-plugin-state.js +73 -0
- package/dist/plugin-distribution/installer.d.ts +89 -0
- package/dist/plugin-distribution/installer.js +395 -0
- package/dist/plugin-distribution/manager.d.ts +128 -0
- package/dist/plugin-distribution/manager.js +358 -0
- package/dist/plugin-distribution/path-safety.d.ts +2 -0
- package/dist/plugin-distribution/path-safety.js +20 -0
- package/dist/plugin-distribution/plugin-bundle-verifier.d.ts +28 -0
- package/dist/plugin-distribution/plugin-bundle-verifier.js +96 -0
- package/dist/plugin-distribution/plugin-bundle-worker.d.ts +34 -0
- package/dist/plugin-distribution/plugin-bundle-worker.js +99 -0
- package/dist/plugin-distribution/plugin-bundle.d.ts +16 -0
- package/dist/plugin-distribution/plugin-bundle.js +244 -0
- package/dist/plugin-distribution/plugin-bundle.worker.d.ts +1 -0
- package/dist/plugin-distribution/plugin-bundle.worker.js +38 -0
- package/dist/plugin-distribution/registry-cache.d.ts +26 -0
- package/dist/plugin-distribution/registry-cache.js +85 -0
- package/dist/plugin-distribution/registry-client.d.ts +36 -0
- package/dist/plugin-distribution/registry-client.js +115 -0
- package/dist/plugin-distribution/reserved-ids.d.ts +4 -0
- package/dist/plugin-distribution/reserved-ids.js +10 -0
- package/dist/plugin-distribution/runtime-validation.d.ts +14 -0
- package/dist/plugin-distribution/runtime-validation.js +388 -0
- package/dist/plugin-distribution/schemas.d.ts +4895 -0
- package/dist/plugin-distribution/schemas.js +299 -0
- package/dist/plugin-distribution/signing.d.ts +5 -0
- package/dist/plugin-distribution/signing.js +38 -0
- package/dist/plugin-distribution/types.d.ts +217 -0
- package/dist/plugin-distribution/types.js +1 -0
- package/dist/plugin-install-progress.d.ts +15 -0
- package/dist/plugin-install-progress.js +86 -0
- package/dist/plugin-manager.d.ts +306 -0
- package/dist/plugin-manager.js +2848 -0
- package/dist/plugin-runtime-entry.d.ts +19 -0
- package/dist/plugin-runtime-entry.js +135 -0
- package/dist/plugin.d.ts +350 -0
- package/dist/plugin.js +754 -0
- package/dist/popover.d.ts +20 -0
- package/dist/popover.js +32 -0
- package/dist/promise-worker.d.ts +22 -0
- package/dist/promise-worker.js +65 -0
- package/dist/prompt-confirm.d.ts +9 -0
- package/dist/prompt-confirm.js +33 -0
- package/dist/search-document-provider.d.ts +45 -0
- package/dist/search-document-provider.js +54 -0
- package/dist/search-query/index.d.ts +78 -0
- package/dist/search-query/index.js +426 -0
- package/dist/search-query/parser.d.ts +3 -0
- package/dist/search-query/parser.js +16 -0
- package/dist/search-query/parser.terms.js +26 -0
- package/dist/search-query/search-query.grammar +128 -0
- package/dist/search.d.ts +19 -0
- package/dist/search.js +104 -0
- package/dist/setting-section-layout.d.ts +19 -0
- package/dist/setting-section-layout.js +194 -0
- package/dist/settings.svelte.d.ts +474 -0
- package/dist/settings.svelte.js +1421 -0
- package/dist/source-text-file-view.d.ts +25 -0
- package/dist/source-text-file-view.js +78 -0
- package/dist/status-bar.svelte.d.ts +29 -0
- package/dist/status-bar.svelte.js +72 -0
- package/dist/storage/app-database.d.ts +860 -0
- package/dist/storage/app-database.js +2372 -0
- package/dist/storage/blob-store.d.ts +12 -0
- package/dist/storage/blob-store.js +1 -0
- package/dist/storage/browser-app-database-coordination.d.ts +35 -0
- package/dist/storage/browser-app-database-coordination.js +207 -0
- package/dist/storage/browser-coordinated-app-database.d.ts +109 -0
- package/dist/storage/browser-coordinated-app-database.js +704 -0
- package/dist/storage/browser.d.ts +153 -0
- package/dist/storage/browser.js +794 -0
- package/dist/storage/desktop-native.d.ts +184 -0
- package/dist/storage/desktop-native.js +415 -0
- package/dist/storage/fs.d.ts +188 -0
- package/dist/storage/fs.js +307 -0
- package/dist/storage/index-projection.d.ts +236 -0
- package/dist/storage/index-projection.js +402 -0
- package/dist/storage/index.d.ts +17 -0
- package/dist/storage/index.js +17 -0
- package/dist/storage/keyval.d.ts +94 -0
- package/dist/storage/keyval.js +199 -0
- package/dist/storage/memory-vault-adapter.d.ts +44 -0
- package/dist/storage/memory-vault-adapter.js +351 -0
- package/dist/storage/native-vault-bootstrap-kv.d.ts +22 -0
- package/dist/storage/native-vault-bootstrap-kv.js +96 -0
- package/dist/storage/path.d.ts +22 -0
- package/dist/storage/path.js +110 -0
- package/dist/storage/search-embedding-provider.d.ts +75 -0
- package/dist/storage/search-embedding-provider.js +446 -0
- package/dist/storage/search-embedding-worker.d.ts +1 -0
- package/dist/storage/search-embedding-worker.js +63 -0
- package/dist/storage/search-query-evaluator.d.ts +14 -0
- package/dist/storage/search-query-evaluator.js +678 -0
- package/dist/storage/task-projection.d.ts +152 -0
- package/dist/storage/task-projection.js +154 -0
- package/dist/storage/turso-app-database.d.ts +150 -0
- package/dist/storage/turso-app-database.js +2671 -0
- package/dist/storage/vault-session.d.ts +37 -0
- package/dist/storage/vault-session.js +196 -0
- package/dist/storage/vault-state.d.ts +74 -0
- package/dist/storage/vault-state.js +181 -0
- package/dist/storage/vault.svelte.d.ts +257 -0
- package/dist/storage/vault.svelte.js +661 -0
- package/dist/suggest.d.ts +193 -0
- package/dist/suggest.js +391 -0
- package/dist/tasks.d.ts +7 -0
- package/dist/tasks.js +18 -0
- package/dist/telemetry.d.ts +116 -0
- package/dist/telemetry.js +103 -0
- package/dist/utils.d.ts +55 -0
- package/dist/utils.js +314 -0
- package/dist/vault-api.d.ts +5 -0
- package/dist/vault-api.js +3 -0
- package/dist/vault-maintenance-commands.d.ts +5 -0
- package/dist/vault-maintenance-commands.js +32 -0
- package/dist/view.svelte.d.ts +215 -0
- package/dist/view.svelte.js +381 -0
- package/dist/vite-env.d.ts +2 -0
- package/dist/workspace-host-internal.d.ts +7 -0
- package/dist/workspace-host-internal.js +7 -0
- package/dist/workspace-host.d.ts +10 -0
- package/dist/workspace-host.js +14 -0
- package/dist/workspace-layout-commands.d.ts +3 -0
- package/dist/workspace-layout-commands.js +72 -0
- package/dist/workspace-layout-normalizer.d.ts +92 -0
- package/dist/workspace-layout-normalizer.js +301 -0
- package/dist/workspace-layouts.d.ts +7 -0
- package/dist/workspace-layouts.js +23 -0
- package/dist/workspace-trust.d.ts +32 -0
- package/dist/workspace-trust.js +72 -0
- package/dist/workspace.svelte.d.ts +796 -0
- package/dist/workspace.svelte.js +4380 -0
- package/package.json +240 -0
|
@@ -0,0 +1,2671 @@
|
|
|
1
|
+
import { collectSearchQueryTerms } from "../search-query";
|
|
2
|
+
import { buildSearchResult, compareSearchResults, embeddingErrorState, EMPTY_APP_DATABASE_CAPABILITIES, hasSearchPropertyNames, MIN_VECTOR_SEARCH_SCORE, normalizeSearchDocument, pathWithinPrefix, rankSearchScores, resolveSearchQueryEnhancementDiagnostics, scoreSearchDocument, scoreVectorDocument, searchDocumentProperties, searchPropertyNames, } from "./app-database";
|
|
3
|
+
import { assertProjectionReadAccess, assertProjectionWriteAccess, compileProjectionQuerySql, evaluateProjectionQuery, indexedValuesForRow, MAX_PROJECTION_ROWS_PER_SOURCE, projectionIndexStatus, PUBLIC_TASKS_PROJECTION_ID, } from "./index-projection";
|
|
4
|
+
import { TASK_PROJECTION_FIELDS, TASK_PROJECTION_VERSION, taskQueryToIndexQuery, } from "./task-projection";
|
|
5
|
+
import { createSearchEmbeddingProvider, } from "./search-embedding-provider";
|
|
6
|
+
import { isStructuredSearchQuery } from "./search-query-evaluator";
|
|
7
|
+
export const TURSO_APP_DATABASE_SCHEMA_VERSION = 5;
|
|
8
|
+
export const TURSO_APP_DATABASE_SCHEMA = `
|
|
9
|
+
CREATE TABLE IF NOT EXISTS schema_meta (
|
|
10
|
+
key TEXT PRIMARY KEY,
|
|
11
|
+
value TEXT NOT NULL
|
|
12
|
+
);
|
|
13
|
+
CREATE TABLE IF NOT EXISTS app_state (
|
|
14
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
15
|
+
state_json TEXT NOT NULL
|
|
16
|
+
);
|
|
17
|
+
CREATE TABLE IF NOT EXISTS app_meta (
|
|
18
|
+
key TEXT PRIMARY KEY,
|
|
19
|
+
value_json TEXT NOT NULL
|
|
20
|
+
);
|
|
21
|
+
CREATE TABLE IF NOT EXISTS app_revision (
|
|
22
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
23
|
+
revision INTEGER NOT NULL
|
|
24
|
+
);
|
|
25
|
+
CREATE TABLE IF NOT EXISTS app_changes (
|
|
26
|
+
revision INTEGER PRIMARY KEY,
|
|
27
|
+
domains_json TEXT NOT NULL,
|
|
28
|
+
paths_json TEXT NOT NULL,
|
|
29
|
+
renamed_json TEXT,
|
|
30
|
+
committed_at INTEGER NOT NULL
|
|
31
|
+
);
|
|
32
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
33
|
+
path TEXT PRIMARY KEY,
|
|
34
|
+
normalized_path TEXT NOT NULL,
|
|
35
|
+
extension TEXT NOT NULL,
|
|
36
|
+
mtime INTEGER NOT NULL,
|
|
37
|
+
size INTEGER NOT NULL,
|
|
38
|
+
hash TEXT NOT NULL,
|
|
39
|
+
indexed INTEGER NOT NULL,
|
|
40
|
+
deleted INTEGER NOT NULL
|
|
41
|
+
);
|
|
42
|
+
CREATE INDEX IF NOT EXISTS files_manifest_idx
|
|
43
|
+
ON files(deleted, indexed, path);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS files_extension_path_idx
|
|
45
|
+
ON files(extension, deleted, indexed, path);
|
|
46
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
47
|
+
path TEXT PRIMARY KEY,
|
|
48
|
+
hash TEXT NOT NULL,
|
|
49
|
+
parser_version TEXT NOT NULL,
|
|
50
|
+
data_json TEXT NOT NULL
|
|
51
|
+
);
|
|
52
|
+
CREATE TABLE IF NOT EXISTS links (
|
|
53
|
+
source_path TEXT NOT NULL,
|
|
54
|
+
ordinal INTEGER NOT NULL,
|
|
55
|
+
data_json TEXT NOT NULL,
|
|
56
|
+
PRIMARY KEY (source_path, ordinal)
|
|
57
|
+
);
|
|
58
|
+
CREATE TABLE IF NOT EXISTS tags (
|
|
59
|
+
path TEXT NOT NULL,
|
|
60
|
+
ordinal INTEGER NOT NULL,
|
|
61
|
+
data_json TEXT NOT NULL,
|
|
62
|
+
PRIMARY KEY (path, ordinal)
|
|
63
|
+
);
|
|
64
|
+
CREATE TABLE IF NOT EXISTS properties (
|
|
65
|
+
path TEXT NOT NULL,
|
|
66
|
+
ordinal INTEGER NOT NULL,
|
|
67
|
+
data_json TEXT NOT NULL,
|
|
68
|
+
PRIMARY KEY (path, ordinal)
|
|
69
|
+
);
|
|
70
|
+
CREATE TABLE IF NOT EXISTS metadata_links (
|
|
71
|
+
source_path TEXT NOT NULL,
|
|
72
|
+
ordinal INTEGER NOT NULL,
|
|
73
|
+
normalized_target TEXT NOT NULL,
|
|
74
|
+
resolved_target_path TEXT,
|
|
75
|
+
resolution_state TEXT NOT NULL,
|
|
76
|
+
link_type TEXT NOT NULL,
|
|
77
|
+
link_kind TEXT,
|
|
78
|
+
data_json TEXT NOT NULL,
|
|
79
|
+
PRIMARY KEY (source_path, ordinal)
|
|
80
|
+
);
|
|
81
|
+
CREATE INDEX IF NOT EXISTS metadata_links_incoming_idx
|
|
82
|
+
ON metadata_links(resolved_target_path, source_path, ordinal);
|
|
83
|
+
CREATE INDEX IF NOT EXISTS metadata_links_unresolved_idx
|
|
84
|
+
ON metadata_links(resolution_state, normalized_target, source_path);
|
|
85
|
+
CREATE TABLE IF NOT EXISTS metadata_tags (
|
|
86
|
+
path TEXT NOT NULL,
|
|
87
|
+
ordinal INTEGER NOT NULL,
|
|
88
|
+
tag TEXT NOT NULL,
|
|
89
|
+
normalized_tag TEXT NOT NULL,
|
|
90
|
+
data_json TEXT NOT NULL,
|
|
91
|
+
PRIMARY KEY (path, ordinal)
|
|
92
|
+
);
|
|
93
|
+
CREATE INDEX IF NOT EXISTS metadata_tags_exact_idx
|
|
94
|
+
ON metadata_tags(normalized_tag, path);
|
|
95
|
+
CREATE TABLE IF NOT EXISTS metadata_tag_ancestors (
|
|
96
|
+
path TEXT NOT NULL,
|
|
97
|
+
tag_ordinal INTEGER NOT NULL,
|
|
98
|
+
ancestor TEXT NOT NULL,
|
|
99
|
+
depth INTEGER NOT NULL,
|
|
100
|
+
PRIMARY KEY (path, tag_ordinal, ancestor)
|
|
101
|
+
);
|
|
102
|
+
CREATE INDEX IF NOT EXISTS metadata_tag_ancestors_idx
|
|
103
|
+
ON metadata_tag_ancestors(ancestor, path);
|
|
104
|
+
CREATE TABLE IF NOT EXISTS metadata_properties (
|
|
105
|
+
path TEXT NOT NULL,
|
|
106
|
+
ordinal INTEGER NOT NULL,
|
|
107
|
+
name TEXT NOT NULL,
|
|
108
|
+
normalized_name TEXT NOT NULL,
|
|
109
|
+
inferred_type TEXT NOT NULL,
|
|
110
|
+
declared_type TEXT,
|
|
111
|
+
data_json TEXT NOT NULL,
|
|
112
|
+
PRIMARY KEY (path, ordinal)
|
|
113
|
+
);
|
|
114
|
+
CREATE INDEX IF NOT EXISTS metadata_properties_name_idx
|
|
115
|
+
ON metadata_properties(normalized_name, path);
|
|
116
|
+
CREATE TABLE IF NOT EXISTS metadata_property_values (
|
|
117
|
+
path TEXT NOT NULL,
|
|
118
|
+
property_ordinal INTEGER NOT NULL,
|
|
119
|
+
value_ordinal INTEGER NOT NULL,
|
|
120
|
+
property_name TEXT NOT NULL,
|
|
121
|
+
property_path TEXT NOT NULL,
|
|
122
|
+
normalized_property_path TEXT NOT NULL,
|
|
123
|
+
value_type TEXT NOT NULL,
|
|
124
|
+
text_value TEXT,
|
|
125
|
+
number_value REAL,
|
|
126
|
+
boolean_value INTEGER,
|
|
127
|
+
date_value TEXT,
|
|
128
|
+
data_json TEXT NOT NULL,
|
|
129
|
+
PRIMARY KEY (path, property_ordinal, value_ordinal, property_path)
|
|
130
|
+
);
|
|
131
|
+
CREATE INDEX IF NOT EXISTS metadata_property_text_idx
|
|
132
|
+
ON metadata_property_values(property_name, text_value, path);
|
|
133
|
+
CREATE INDEX IF NOT EXISTS metadata_property_number_idx
|
|
134
|
+
ON metadata_property_values(property_name, number_value, path);
|
|
135
|
+
CREATE INDEX IF NOT EXISTS metadata_property_boolean_idx
|
|
136
|
+
ON metadata_property_values(property_name, boolean_value, path);
|
|
137
|
+
CREATE INDEX IF NOT EXISTS metadata_property_date_idx
|
|
138
|
+
ON metadata_property_values(property_name, date_value, path);
|
|
139
|
+
CREATE TABLE IF NOT EXISTS tasks (
|
|
140
|
+
document_path TEXT PRIMARY KEY,
|
|
141
|
+
document_id TEXT NOT NULL,
|
|
142
|
+
kind TEXT NOT NULL,
|
|
143
|
+
title TEXT NOT NULL,
|
|
144
|
+
status TEXT NOT NULL,
|
|
145
|
+
inbox INTEGER NOT NULL DEFAULT 0,
|
|
146
|
+
start_kind TEXT NOT NULL,
|
|
147
|
+
start_date TEXT,
|
|
148
|
+
plan_date TEXT,
|
|
149
|
+
plan_kind TEXT,
|
|
150
|
+
plan_time TEXT,
|
|
151
|
+
duration_minutes INTEGER,
|
|
152
|
+
deadline TEXT,
|
|
153
|
+
completed_at TEXT,
|
|
154
|
+
repeat_strategy TEXT,
|
|
155
|
+
repeat_frequency TEXT,
|
|
156
|
+
repeat_interval INTEGER,
|
|
157
|
+
repeat_anchor TEXT,
|
|
158
|
+
checklist_total INTEGER NOT NULL DEFAULT 0,
|
|
159
|
+
checklist_completed INTEGER NOT NULL DEFAULT 0,
|
|
160
|
+
comment_count INTEGER NOT NULL DEFAULT 0,
|
|
161
|
+
projection_version INTEGER NOT NULL
|
|
162
|
+
);
|
|
163
|
+
CREATE TABLE IF NOT EXISTS task_records (
|
|
164
|
+
document_path TEXT PRIMARY KEY,
|
|
165
|
+
data_json TEXT NOT NULL
|
|
166
|
+
);
|
|
167
|
+
CREATE INDEX IF NOT EXISTS tasks_open_plan ON tasks(status, plan_date);
|
|
168
|
+
CREATE INDEX IF NOT EXISTS tasks_open_deadline ON tasks(status, deadline);
|
|
169
|
+
CREATE INDEX IF NOT EXISTS tasks_open_start ON tasks(status, start_kind, start_date);
|
|
170
|
+
CREATE INDEX IF NOT EXISTS tasks_inbox ON tasks(status, inbox);
|
|
171
|
+
CREATE TABLE IF NOT EXISTS index_projections (
|
|
172
|
+
projection_id TEXT PRIMARY KEY,
|
|
173
|
+
owner_plugin_id TEXT NOT NULL,
|
|
174
|
+
schema_version INTEGER NOT NULL,
|
|
175
|
+
config_hash TEXT NOT NULL,
|
|
176
|
+
visibility TEXT NOT NULL,
|
|
177
|
+
fields_json TEXT NOT NULL,
|
|
178
|
+
active INTEGER NOT NULL,
|
|
179
|
+
updated_at INTEGER NOT NULL
|
|
180
|
+
);
|
|
181
|
+
CREATE TABLE IF NOT EXISTS index_projection_sources (
|
|
182
|
+
projection_id TEXT NOT NULL,
|
|
183
|
+
source_path TEXT NOT NULL,
|
|
184
|
+
source_hash TEXT NOT NULL,
|
|
185
|
+
schema_version INTEGER NOT NULL,
|
|
186
|
+
config_hash TEXT NOT NULL,
|
|
187
|
+
status TEXT NOT NULL,
|
|
188
|
+
error TEXT,
|
|
189
|
+
indexed_at INTEGER NOT NULL,
|
|
190
|
+
PRIMARY KEY (projection_id, source_path)
|
|
191
|
+
);
|
|
192
|
+
CREATE TABLE IF NOT EXISTS index_projection_rows (
|
|
193
|
+
projection_id TEXT NOT NULL,
|
|
194
|
+
row_id TEXT NOT NULL,
|
|
195
|
+
source_path TEXT NOT NULL,
|
|
196
|
+
kind TEXT NOT NULL,
|
|
197
|
+
ordinal INTEGER NOT NULL DEFAULT 0,
|
|
198
|
+
data_json TEXT NOT NULL,
|
|
199
|
+
PRIMARY KEY (projection_id, row_id)
|
|
200
|
+
);
|
|
201
|
+
CREATE INDEX IF NOT EXISTS index_projection_rows_source
|
|
202
|
+
ON index_projection_rows(projection_id, source_path);
|
|
203
|
+
CREATE TABLE IF NOT EXISTS index_projection_values (
|
|
204
|
+
projection_id TEXT NOT NULL,
|
|
205
|
+
row_id TEXT NOT NULL,
|
|
206
|
+
field TEXT NOT NULL,
|
|
207
|
+
ordinal INTEGER NOT NULL DEFAULT 0,
|
|
208
|
+
value_type TEXT NOT NULL,
|
|
209
|
+
text_value TEXT,
|
|
210
|
+
number_value REAL,
|
|
211
|
+
boolean_value INTEGER,
|
|
212
|
+
date_value TEXT,
|
|
213
|
+
datetime_value INTEGER,
|
|
214
|
+
PRIMARY KEY (projection_id, row_id, field, ordinal)
|
|
215
|
+
);
|
|
216
|
+
CREATE TABLE IF NOT EXISTS index_projection_edges (
|
|
217
|
+
projection_id TEXT NOT NULL,
|
|
218
|
+
source_row_id TEXT NOT NULL,
|
|
219
|
+
relation TEXT NOT NULL,
|
|
220
|
+
target_projection_id TEXT,
|
|
221
|
+
target_row_id TEXT,
|
|
222
|
+
target_path TEXT,
|
|
223
|
+
target_text TEXT,
|
|
224
|
+
ordinal INTEGER NOT NULL,
|
|
225
|
+
data_json TEXT,
|
|
226
|
+
PRIMARY KEY (projection_id, source_row_id, relation, ordinal)
|
|
227
|
+
);
|
|
228
|
+
CREATE TABLE IF NOT EXISTS history_files (
|
|
229
|
+
file_id TEXT PRIMARY KEY,
|
|
230
|
+
data_json TEXT NOT NULL
|
|
231
|
+
);
|
|
232
|
+
CREATE TABLE IF NOT EXISTS history_file_paths (
|
|
233
|
+
path TEXT PRIMARY KEY,
|
|
234
|
+
file_id TEXT NOT NULL
|
|
235
|
+
);
|
|
236
|
+
CREATE TABLE IF NOT EXISTS history_revisions (
|
|
237
|
+
file_id TEXT NOT NULL,
|
|
238
|
+
ordinal INTEGER NOT NULL,
|
|
239
|
+
data_json TEXT NOT NULL,
|
|
240
|
+
PRIMARY KEY (file_id, ordinal)
|
|
241
|
+
);
|
|
242
|
+
CREATE TABLE IF NOT EXISTS notifications (
|
|
243
|
+
id TEXT PRIMARY KEY,
|
|
244
|
+
data_json TEXT NOT NULL
|
|
245
|
+
);
|
|
246
|
+
CREATE TABLE IF NOT EXISTS search_docs (
|
|
247
|
+
path TEXT PRIMARY KEY,
|
|
248
|
+
source_provider_id TEXT,
|
|
249
|
+
metadata_hash TEXT,
|
|
250
|
+
provider_version TEXT,
|
|
251
|
+
projection_signature TEXT,
|
|
252
|
+
source_mtime INTEGER,
|
|
253
|
+
source_size INTEGER,
|
|
254
|
+
name TEXT NOT NULL,
|
|
255
|
+
extension TEXT NOT NULL,
|
|
256
|
+
checksum TEXT NOT NULL,
|
|
257
|
+
content TEXT NOT NULL,
|
|
258
|
+
tags TEXT NOT NULL,
|
|
259
|
+
metadata_text TEXT NOT NULL,
|
|
260
|
+
data_json TEXT NOT NULL
|
|
261
|
+
);
|
|
262
|
+
CREATE TABLE IF NOT EXISTS search_chunks (
|
|
263
|
+
id TEXT PRIMARY KEY,
|
|
264
|
+
path TEXT NOT NULL,
|
|
265
|
+
text TEXT NOT NULL,
|
|
266
|
+
embedding_json TEXT,
|
|
267
|
+
embedding BLOB
|
|
268
|
+
);
|
|
269
|
+
CREATE INDEX IF NOT EXISTS search_chunks_path_idx ON search_chunks(path);
|
|
270
|
+
CREATE TABLE IF NOT EXISTS memory_source_state (
|
|
271
|
+
source_key TEXT PRIMARY KEY,
|
|
272
|
+
source_path TEXT NOT NULL,
|
|
273
|
+
source_hash TEXT NOT NULL,
|
|
274
|
+
status TEXT NOT NULL,
|
|
275
|
+
indexed_at INTEGER NOT NULL,
|
|
276
|
+
data_json TEXT NOT NULL
|
|
277
|
+
);
|
|
278
|
+
CREATE TABLE IF NOT EXISTS memory_candidates (
|
|
279
|
+
id TEXT PRIMARY KEY,
|
|
280
|
+
scope_kind TEXT NOT NULL,
|
|
281
|
+
scope_path TEXT NOT NULL,
|
|
282
|
+
state TEXT NOT NULL,
|
|
283
|
+
last_seen_at INTEGER NOT NULL,
|
|
284
|
+
data_json TEXT NOT NULL
|
|
285
|
+
);
|
|
286
|
+
CREATE INDEX IF NOT EXISTS memory_candidates_scope_state
|
|
287
|
+
ON memory_candidates(scope_kind, scope_path, state, last_seen_at DESC);
|
|
288
|
+
CREATE TABLE IF NOT EXISTS memory_candidate_origins (
|
|
289
|
+
candidate_id TEXT NOT NULL,
|
|
290
|
+
conversation_id TEXT NOT NULL,
|
|
291
|
+
entry_id TEXT NOT NULL,
|
|
292
|
+
data_json TEXT NOT NULL,
|
|
293
|
+
PRIMARY KEY (candidate_id, conversation_id, entry_id)
|
|
294
|
+
);
|
|
295
|
+
CREATE INDEX IF NOT EXISTS memory_candidate_origins_conversation
|
|
296
|
+
ON memory_candidate_origins(conversation_id, candidate_id);
|
|
297
|
+
CREATE TABLE IF NOT EXISTS memory_recall_signals (
|
|
298
|
+
target_ref TEXT NOT NULL,
|
|
299
|
+
query_fingerprint TEXT NOT NULL,
|
|
300
|
+
day TEXT NOT NULL,
|
|
301
|
+
best_score REAL NOT NULL,
|
|
302
|
+
hit_count INTEGER NOT NULL,
|
|
303
|
+
data_json TEXT NOT NULL,
|
|
304
|
+
PRIMARY KEY (target_ref, query_fingerprint, day)
|
|
305
|
+
);
|
|
306
|
+
CREATE TABLE IF NOT EXISTS memory_jobs (
|
|
307
|
+
id TEXT PRIMARY KEY,
|
|
308
|
+
scope_key TEXT NOT NULL,
|
|
309
|
+
status TEXT NOT NULL,
|
|
310
|
+
lease_until INTEGER,
|
|
311
|
+
created_at INTEGER NOT NULL,
|
|
312
|
+
data_json TEXT NOT NULL
|
|
313
|
+
);
|
|
314
|
+
CREATE INDEX IF NOT EXISTS memory_jobs_scope_created
|
|
315
|
+
ON memory_jobs(scope_key, created_at DESC);
|
|
316
|
+
`;
|
|
317
|
+
const TURSO_SEARCH_FTS_SCHEMA = `
|
|
318
|
+
CREATE INDEX IF NOT EXISTS search_docs_fts ON search_docs USING fts (
|
|
319
|
+
name,
|
|
320
|
+
path,
|
|
321
|
+
content,
|
|
322
|
+
tags,
|
|
323
|
+
metadata_text
|
|
324
|
+
) WITH (weights = 'name=2.0,path=1.5,content=1.0,tags=1.5,metadata_text=1.0');
|
|
325
|
+
`;
|
|
326
|
+
function clone(value) {
|
|
327
|
+
if (value === undefined || value === null)
|
|
328
|
+
return value;
|
|
329
|
+
return structuredClone(value);
|
|
330
|
+
}
|
|
331
|
+
function stableVaultHash(value) {
|
|
332
|
+
let hash = 2166136261;
|
|
333
|
+
for (const character of value) {
|
|
334
|
+
hash ^= character.codePointAt(0) ?? 0;
|
|
335
|
+
hash = Math.imul(hash, 16777619);
|
|
336
|
+
}
|
|
337
|
+
return (hash >>> 0).toString(16).padStart(8, "0");
|
|
338
|
+
}
|
|
339
|
+
export function tursoWasmDatabasePath(vaultId) {
|
|
340
|
+
return `lapis-app-${stableVaultHash(vaultId)}.turso`;
|
|
341
|
+
}
|
|
342
|
+
function ftsQueryForTerms(terms) {
|
|
343
|
+
return terms
|
|
344
|
+
.map((term) => `"${term.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`)
|
|
345
|
+
.join(" AND ");
|
|
346
|
+
}
|
|
347
|
+
const METADATA_SNAPSHOT_META_KEY = "compat.metadataSnapshot";
|
|
348
|
+
const SEARCH_EMBEDDING_PROVIDER_META_KEY = "search.embeddingProvider";
|
|
349
|
+
const MAX_BOUND_PATHS_PER_QUERY = 400;
|
|
350
|
+
function chunkPaths(paths) {
|
|
351
|
+
const chunks = [];
|
|
352
|
+
for (let index = 0; index < paths.length; index += MAX_BOUND_PATHS_PER_QUERY) {
|
|
353
|
+
chunks.push(paths.slice(index, index + MAX_BOUND_PATHS_PER_QUERY));
|
|
354
|
+
}
|
|
355
|
+
return chunks;
|
|
356
|
+
}
|
|
357
|
+
function balancedSqlDisjunction(clauses) {
|
|
358
|
+
if (!clauses.length)
|
|
359
|
+
return "0";
|
|
360
|
+
let level = [...clauses];
|
|
361
|
+
while (level.length > 1) {
|
|
362
|
+
const next = [];
|
|
363
|
+
for (let index = 0; index < level.length; index += 2) {
|
|
364
|
+
const left = level[index];
|
|
365
|
+
const right = level[index + 1];
|
|
366
|
+
next.push(right ? `(${left} OR ${right})` : left);
|
|
367
|
+
}
|
|
368
|
+
level = next;
|
|
369
|
+
}
|
|
370
|
+
return level[0];
|
|
371
|
+
}
|
|
372
|
+
function parseJson(value, fallback) {
|
|
373
|
+
if (typeof value !== "string")
|
|
374
|
+
return fallback;
|
|
375
|
+
try {
|
|
376
|
+
return JSON.parse(value);
|
|
377
|
+
}
|
|
378
|
+
catch {
|
|
379
|
+
return fallback;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
function notebookStateMetaKey(sourcePath) {
|
|
383
|
+
return `notebook.state:${sourcePath}`;
|
|
384
|
+
}
|
|
385
|
+
function createStableId(prefix) {
|
|
386
|
+
if (typeof crypto !== "undefined" &&
|
|
387
|
+
typeof crypto.randomUUID === "function") {
|
|
388
|
+
return `${prefix}:${crypto.randomUUID()}`;
|
|
389
|
+
}
|
|
390
|
+
return `${prefix}:${Date.now().toString(36)}:${Math.random().toString(36).slice(2)}`;
|
|
391
|
+
}
|
|
392
|
+
function hashText(value) {
|
|
393
|
+
let hash = 2166136261;
|
|
394
|
+
for (const character of value) {
|
|
395
|
+
hash ^= character.codePointAt(0) ?? 0;
|
|
396
|
+
hash = Math.imul(hash, 16777619);
|
|
397
|
+
}
|
|
398
|
+
return (hash >>> 0).toString(16).padStart(8, "0");
|
|
399
|
+
}
|
|
400
|
+
function normalizeTag(value) {
|
|
401
|
+
return value.trim().replace(/^#/, "").toLowerCase();
|
|
402
|
+
}
|
|
403
|
+
function normalizeExtension(value) {
|
|
404
|
+
return value.trim().replace(/^\./, "").toLowerCase();
|
|
405
|
+
}
|
|
406
|
+
function isDateScalar(value) {
|
|
407
|
+
return /^\d{4}-\d{2}-\d{2}(?:$|T)/.test(value);
|
|
408
|
+
}
|
|
409
|
+
function flattenPropertyValues(value, path, output = []) {
|
|
410
|
+
if (Array.isArray(value)) {
|
|
411
|
+
value.forEach((entry, index) => flattenPropertyValues(entry, `${path}[${index}]`, output));
|
|
412
|
+
return output;
|
|
413
|
+
}
|
|
414
|
+
if (value && typeof value === "object") {
|
|
415
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
416
|
+
flattenPropertyValues(entry, path ? `${path}.${key}` : key, output);
|
|
417
|
+
}
|
|
418
|
+
return output;
|
|
419
|
+
}
|
|
420
|
+
output.push({ path, value });
|
|
421
|
+
return output;
|
|
422
|
+
}
|
|
423
|
+
function normalizePropertyPath(path) {
|
|
424
|
+
return path.toLowerCase().replace(/\[\d+\]/gu, "[]");
|
|
425
|
+
}
|
|
426
|
+
function indexedFileStatements(record) {
|
|
427
|
+
const path = record.file.path;
|
|
428
|
+
const statements = [
|
|
429
|
+
{
|
|
430
|
+
sql: `INSERT INTO files
|
|
431
|
+
(path, normalized_path, extension, mtime, size, hash, indexed, deleted)
|
|
432
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 0)
|
|
433
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
434
|
+
normalized_path = excluded.normalized_path,
|
|
435
|
+
extension = excluded.extension,
|
|
436
|
+
mtime = excluded.mtime,
|
|
437
|
+
size = excluded.size,
|
|
438
|
+
hash = excluded.hash,
|
|
439
|
+
indexed = excluded.indexed,
|
|
440
|
+
deleted = 0`,
|
|
441
|
+
args: [
|
|
442
|
+
path,
|
|
443
|
+
record.file.normalizedPath,
|
|
444
|
+
normalizeExtension(record.file.extension),
|
|
445
|
+
record.file.mtime,
|
|
446
|
+
record.file.size,
|
|
447
|
+
record.file.hash,
|
|
448
|
+
record.file.indexed ? 1 : 0,
|
|
449
|
+
],
|
|
450
|
+
},
|
|
451
|
+
{
|
|
452
|
+
sql: `INSERT INTO metadata (path, hash, parser_version, data_json)
|
|
453
|
+
VALUES (?, ?, ?, ?)
|
|
454
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
455
|
+
hash = excluded.hash,
|
|
456
|
+
parser_version = excluded.parser_version,
|
|
457
|
+
data_json = excluded.data_json`,
|
|
458
|
+
args: [
|
|
459
|
+
record.metadata.path,
|
|
460
|
+
record.metadata.hash,
|
|
461
|
+
record.metadata.parserVersion,
|
|
462
|
+
JSON.stringify(record.metadata.metadata),
|
|
463
|
+
],
|
|
464
|
+
},
|
|
465
|
+
{ sql: "DELETE FROM links WHERE source_path = ?", args: [path] },
|
|
466
|
+
{ sql: "DELETE FROM tags WHERE path = ?", args: [path] },
|
|
467
|
+
{ sql: "DELETE FROM properties WHERE path = ?", args: [path] },
|
|
468
|
+
{ sql: "DELETE FROM metadata_links WHERE source_path = ?", args: [path] },
|
|
469
|
+
{ sql: "DELETE FROM metadata_tags WHERE path = ?", args: [path] },
|
|
470
|
+
{ sql: "DELETE FROM metadata_tag_ancestors WHERE path = ?", args: [path] },
|
|
471
|
+
{ sql: "DELETE FROM metadata_properties WHERE path = ?", args: [path] },
|
|
472
|
+
{
|
|
473
|
+
sql: "DELETE FROM metadata_property_values WHERE path = ?",
|
|
474
|
+
args: [path],
|
|
475
|
+
},
|
|
476
|
+
];
|
|
477
|
+
record.links.forEach((link, ordinal) => {
|
|
478
|
+
const data = { ...link, ordinal: link.ordinal ?? ordinal };
|
|
479
|
+
statements.push({
|
|
480
|
+
sql: "INSERT INTO links (source_path, ordinal, data_json) VALUES (?, ?, ?)",
|
|
481
|
+
args: [path, ordinal, JSON.stringify(data)],
|
|
482
|
+
}, {
|
|
483
|
+
sql: `INSERT INTO metadata_links
|
|
484
|
+
(source_path, ordinal, normalized_target, resolved_target_path, resolution_state, link_type, link_kind, data_json)
|
|
485
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
486
|
+
args: [
|
|
487
|
+
path,
|
|
488
|
+
ordinal,
|
|
489
|
+
link.targetText.trim().toLowerCase(),
|
|
490
|
+
link.resolvedTargetPath,
|
|
491
|
+
link.resolvedTargetPath ? "resolved" : "unresolved",
|
|
492
|
+
link.type,
|
|
493
|
+
link.kind ?? null,
|
|
494
|
+
JSON.stringify(data),
|
|
495
|
+
],
|
|
496
|
+
});
|
|
497
|
+
});
|
|
498
|
+
record.tags.forEach((tag, ordinal) => {
|
|
499
|
+
statements.push({
|
|
500
|
+
sql: "INSERT INTO tags (path, ordinal, data_json) VALUES (?, ?, ?)",
|
|
501
|
+
args: [path, ordinal, JSON.stringify(tag)],
|
|
502
|
+
}, {
|
|
503
|
+
sql: `INSERT INTO metadata_tags
|
|
504
|
+
(path, ordinal, tag, normalized_tag, data_json)
|
|
505
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
506
|
+
args: [
|
|
507
|
+
path,
|
|
508
|
+
ordinal,
|
|
509
|
+
tag.tag,
|
|
510
|
+
normalizeTag(tag.tag),
|
|
511
|
+
JSON.stringify(tag),
|
|
512
|
+
],
|
|
513
|
+
});
|
|
514
|
+
[...new Set(tag.hierarchy.map(normalizeTag))].forEach((ancestor, depth) => {
|
|
515
|
+
statements.push({
|
|
516
|
+
sql: `INSERT INTO metadata_tag_ancestors
|
|
517
|
+
(path, tag_ordinal, ancestor, depth) VALUES (?, ?, ?, ?)`,
|
|
518
|
+
args: [path, ordinal, ancestor, depth],
|
|
519
|
+
});
|
|
520
|
+
});
|
|
521
|
+
});
|
|
522
|
+
record.properties.forEach((property, ordinal) => {
|
|
523
|
+
const normalizedName = property.name.trim().toLowerCase();
|
|
524
|
+
statements.push({
|
|
525
|
+
sql: "INSERT INTO properties (path, ordinal, data_json) VALUES (?, ?, ?)",
|
|
526
|
+
args: [path, ordinal, JSON.stringify(property)],
|
|
527
|
+
}, {
|
|
528
|
+
sql: `INSERT INTO metadata_properties
|
|
529
|
+
(path, ordinal, name, normalized_name, inferred_type, declared_type, data_json)
|
|
530
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
531
|
+
args: [
|
|
532
|
+
path,
|
|
533
|
+
ordinal,
|
|
534
|
+
property.name,
|
|
535
|
+
normalizedName,
|
|
536
|
+
property.inferredType,
|
|
537
|
+
property.declaredType ?? null,
|
|
538
|
+
JSON.stringify(property),
|
|
539
|
+
],
|
|
540
|
+
});
|
|
541
|
+
flattenPropertyValues(property.value, property.name).forEach((entry, valueOrdinal) => {
|
|
542
|
+
const value = entry.value;
|
|
543
|
+
const valueType = value === null || value === undefined
|
|
544
|
+
? "null"
|
|
545
|
+
: typeof value === "number"
|
|
546
|
+
? "number"
|
|
547
|
+
: typeof value === "boolean"
|
|
548
|
+
? "boolean"
|
|
549
|
+
: typeof value === "string" && isDateScalar(value)
|
|
550
|
+
? "date"
|
|
551
|
+
: "string";
|
|
552
|
+
statements.push({
|
|
553
|
+
sql: `INSERT INTO metadata_property_values
|
|
554
|
+
(path, property_ordinal, value_ordinal, property_name, property_path,
|
|
555
|
+
normalized_property_path,
|
|
556
|
+
value_type, text_value, number_value, boolean_value, date_value, data_json)
|
|
557
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
558
|
+
args: [
|
|
559
|
+
path,
|
|
560
|
+
ordinal,
|
|
561
|
+
valueOrdinal,
|
|
562
|
+
normalizedName,
|
|
563
|
+
entry.path,
|
|
564
|
+
normalizePropertyPath(entry.path),
|
|
565
|
+
valueType,
|
|
566
|
+
valueType === "string" ? String(value) : null,
|
|
567
|
+
valueType === "number" ? value : null,
|
|
568
|
+
valueType === "boolean" ? (value ? 1 : 0) : null,
|
|
569
|
+
valueType === "date" ? String(value) : null,
|
|
570
|
+
JSON.stringify(value ?? null),
|
|
571
|
+
],
|
|
572
|
+
});
|
|
573
|
+
});
|
|
574
|
+
});
|
|
575
|
+
return statements;
|
|
576
|
+
}
|
|
577
|
+
export class TursoAppDatabase {
|
|
578
|
+
vaultId;
|
|
579
|
+
options;
|
|
580
|
+
kind;
|
|
581
|
+
connection = null;
|
|
582
|
+
opened = false;
|
|
583
|
+
nativeFullTextSearch = false;
|
|
584
|
+
nativeVectorSearch = false;
|
|
585
|
+
searchEmbeddingProviderConfig = null;
|
|
586
|
+
searchEmbeddingProvider = null;
|
|
587
|
+
currentRevision = 0;
|
|
588
|
+
commitQueue = Promise.resolve();
|
|
589
|
+
changeListeners = new Set();
|
|
590
|
+
constructor(vaultId, options) {
|
|
591
|
+
this.vaultId = vaultId;
|
|
592
|
+
this.options = options;
|
|
593
|
+
this.kind = options.kind;
|
|
594
|
+
}
|
|
595
|
+
get descriptor() {
|
|
596
|
+
return {
|
|
597
|
+
providerId: this.options.providerId ?? "turso-local",
|
|
598
|
+
engine: "turso",
|
|
599
|
+
transport: this.options.transport,
|
|
600
|
+
role: this.options.role ?? "direct",
|
|
601
|
+
storageMode: "local",
|
|
602
|
+
capabilities: {
|
|
603
|
+
...EMPTY_APP_DATABASE_CAPABILITIES,
|
|
604
|
+
nativeFullTextSearch: this.nativeFullTextSearch,
|
|
605
|
+
vectorSearch: this.nativeVectorSearch,
|
|
606
|
+
localEmbeddings: true,
|
|
607
|
+
crossTabCoordination: this.options.crossTabCoordination ?? false,
|
|
608
|
+
},
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
async open() {
|
|
612
|
+
if (this.opened)
|
|
613
|
+
return;
|
|
614
|
+
this.connection = await this.options.connectionFactory();
|
|
615
|
+
await this.migrate();
|
|
616
|
+
const revision = await this.connection.get("SELECT revision FROM app_revision WHERE id = 1");
|
|
617
|
+
this.currentRevision = Number(revision?.revision ?? 0);
|
|
618
|
+
this.searchEmbeddingProviderConfig =
|
|
619
|
+
(await this.getMeta(SEARCH_EMBEDDING_PROVIDER_META_KEY)) ?? null;
|
|
620
|
+
this.searchEmbeddingProvider = createSearchEmbeddingProvider(this.searchEmbeddingProviderConfig);
|
|
621
|
+
this.opened = true;
|
|
622
|
+
}
|
|
623
|
+
async migrate() {
|
|
624
|
+
const connection = this.requireConnection();
|
|
625
|
+
await connection.exec(TURSO_APP_DATABASE_SCHEMA);
|
|
626
|
+
await this.ensureMetadataPropertyPathSchema();
|
|
627
|
+
const previousVersion = Number((await connection.get("SELECT value FROM schema_meta WHERE key = 'schema.version'"))?.value ?? 0);
|
|
628
|
+
if (previousVersion > 0 &&
|
|
629
|
+
previousVersion < TURSO_APP_DATABASE_SCHEMA_VERSION) {
|
|
630
|
+
const invariants = await this.captureMigrationInvariants();
|
|
631
|
+
await this.backfillNormalizedMetadata();
|
|
632
|
+
await connection.run(`UPDATE search_docs SET
|
|
633
|
+
source_provider_id = json_extract(data_json, '$.sourceProviderId'),
|
|
634
|
+
metadata_hash = json_extract(data_json, '$.sourceMetadata.metadataHash'),
|
|
635
|
+
provider_version = json_extract(data_json, '$.sourceMetadata.providerVersion'),
|
|
636
|
+
projection_signature = json_extract(data_json, '$.sourceMetadata.projectionSignature'),
|
|
637
|
+
source_mtime = json_extract(data_json, '$.sourceMetadata.sourceMtime'),
|
|
638
|
+
source_size = json_extract(data_json, '$.sourceMetadata.sourceSize')`);
|
|
639
|
+
await this.validateMigration(invariants);
|
|
640
|
+
}
|
|
641
|
+
await connection.batch([
|
|
642
|
+
`INSERT INTO app_revision (id, revision) VALUES (1, 0)
|
|
643
|
+
ON CONFLICT(id) DO NOTHING`,
|
|
644
|
+
{
|
|
645
|
+
sql: `INSERT INTO schema_meta (key, value) VALUES ('schema.version', ?)
|
|
646
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
647
|
+
args: [String(TURSO_APP_DATABASE_SCHEMA_VERSION)],
|
|
648
|
+
},
|
|
649
|
+
{
|
|
650
|
+
sql: `INSERT INTO app_meta (key, value_json) VALUES ('schema.version', ?)
|
|
651
|
+
ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json`,
|
|
652
|
+
args: [JSON.stringify(TURSO_APP_DATABASE_SCHEMA_VERSION)],
|
|
653
|
+
},
|
|
654
|
+
], "immediate");
|
|
655
|
+
try {
|
|
656
|
+
await connection.exec(TURSO_SEARCH_FTS_SCHEMA);
|
|
657
|
+
await connection.get("SELECT fts_match(name, path, content, tags, metadata_text, '') AS matched FROM search_docs LIMIT 1");
|
|
658
|
+
this.nativeFullTextSearch = true;
|
|
659
|
+
}
|
|
660
|
+
catch {
|
|
661
|
+
this.nativeFullTextSearch = false;
|
|
662
|
+
}
|
|
663
|
+
try {
|
|
664
|
+
const row = await connection.get("SELECT vector_distance_cos(vector32('[1,0]'), vector32('[1,0]')) AS distance");
|
|
665
|
+
this.nativeVectorSearch = Math.abs(Number(row?.distance)) < 0.000001;
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
this.nativeVectorSearch = false;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
async close() {
|
|
672
|
+
if (!this.connection)
|
|
673
|
+
return;
|
|
674
|
+
const connection = this.connection;
|
|
675
|
+
this.connection = null;
|
|
676
|
+
this.opened = false;
|
|
677
|
+
await this.searchEmbeddingProvider?.dispose?.();
|
|
678
|
+
this.searchEmbeddingProvider = null;
|
|
679
|
+
this.changeListeners.clear();
|
|
680
|
+
await connection.close();
|
|
681
|
+
}
|
|
682
|
+
async beginSearchIndexingBatch() { }
|
|
683
|
+
async endSearchIndexingBatch() { }
|
|
684
|
+
async configureSearchEmbeddingProvider(provider) {
|
|
685
|
+
if (JSON.stringify(provider) ===
|
|
686
|
+
JSON.stringify(this.searchEmbeddingProviderConfig))
|
|
687
|
+
return;
|
|
688
|
+
const previous = this.searchEmbeddingProvider;
|
|
689
|
+
this.searchEmbeddingProviderConfig = clone(provider);
|
|
690
|
+
this.searchEmbeddingProvider = createSearchEmbeddingProvider(provider);
|
|
691
|
+
await previous?.dispose?.();
|
|
692
|
+
await this.commit([
|
|
693
|
+
{
|
|
694
|
+
sql: `INSERT INTO app_meta (key, value_json) VALUES (?, ?)
|
|
695
|
+
ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json`,
|
|
696
|
+
args: [SEARCH_EMBEDDING_PROVIDER_META_KEY, JSON.stringify(provider)],
|
|
697
|
+
},
|
|
698
|
+
], ["search"]);
|
|
699
|
+
}
|
|
700
|
+
async getSearchEmbeddingProvider() {
|
|
701
|
+
return clone(this.searchEmbeddingProviderConfig);
|
|
702
|
+
}
|
|
703
|
+
async getSearchEmbeddingRuntimeStatus() {
|
|
704
|
+
return this.searchEmbeddingProvider
|
|
705
|
+
? clone(this.searchEmbeddingProvider.getRuntimeStatus())
|
|
706
|
+
: null;
|
|
707
|
+
}
|
|
708
|
+
async getSearchIndexStats() {
|
|
709
|
+
const rows = await this.requireConnection().all("SELECT embedding_json FROM search_chunks");
|
|
710
|
+
const stats = {
|
|
711
|
+
documentCount: Number((await this.requireConnection().get("SELECT count(*) AS count FROM search_docs"))?.count ?? 0),
|
|
712
|
+
chunkCount: rows.length,
|
|
713
|
+
readyChunkCount: 0,
|
|
714
|
+
pendingChunkCount: 0,
|
|
715
|
+
errorChunkCount: 0,
|
|
716
|
+
lastError: null,
|
|
717
|
+
};
|
|
718
|
+
for (const row of rows) {
|
|
719
|
+
const embedding = parseJson(row.embedding_json, null);
|
|
720
|
+
if (embedding?.status === "ready")
|
|
721
|
+
stats.readyChunkCount += 1;
|
|
722
|
+
else if (embedding?.status === "error") {
|
|
723
|
+
stats.errorChunkCount += 1;
|
|
724
|
+
stats.lastError ??= embedding.error ?? null;
|
|
725
|
+
}
|
|
726
|
+
else
|
|
727
|
+
stats.pendingChunkCount += 1;
|
|
728
|
+
}
|
|
729
|
+
return stats;
|
|
730
|
+
}
|
|
731
|
+
async getMeta(key) {
|
|
732
|
+
const row = await this.requireConnection().get("SELECT value_json FROM app_meta WHERE key = ?", key);
|
|
733
|
+
return row
|
|
734
|
+
? parseJson(row.value_json, undefined)
|
|
735
|
+
: undefined;
|
|
736
|
+
}
|
|
737
|
+
async setMeta(key, value) {
|
|
738
|
+
await this.commit([
|
|
739
|
+
{
|
|
740
|
+
sql: `INSERT INTO app_meta (key, value_json) VALUES (?, ?)
|
|
741
|
+
ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json`,
|
|
742
|
+
args: [key, JSON.stringify(value)],
|
|
743
|
+
},
|
|
744
|
+
], ["meta"]);
|
|
745
|
+
}
|
|
746
|
+
async getNotebookState(sourcePath) {
|
|
747
|
+
return this.getMeta(notebookStateMetaKey(sourcePath));
|
|
748
|
+
}
|
|
749
|
+
async setNotebookState(sourcePath, state) {
|
|
750
|
+
await this.commit([
|
|
751
|
+
{
|
|
752
|
+
sql: `INSERT INTO app_meta (key, value_json) VALUES (?, ?)
|
|
753
|
+
ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json`,
|
|
754
|
+
args: [notebookStateMetaKey(sourcePath), JSON.stringify(state)],
|
|
755
|
+
},
|
|
756
|
+
], ["notebook"], [sourcePath]);
|
|
757
|
+
}
|
|
758
|
+
async deleteNotebookState(sourcePath) {
|
|
759
|
+
await this.commit([
|
|
760
|
+
{
|
|
761
|
+
sql: "DELETE FROM app_meta WHERE key = ?",
|
|
762
|
+
args: [notebookStateMetaKey(sourcePath)],
|
|
763
|
+
},
|
|
764
|
+
], ["notebook"], [sourcePath]);
|
|
765
|
+
}
|
|
766
|
+
async loadMetadataSnapshot() {
|
|
767
|
+
const stored = await this.getMeta(METADATA_SNAPSHOT_META_KEY);
|
|
768
|
+
if (stored)
|
|
769
|
+
return stored;
|
|
770
|
+
const legacy = await this.requireConnection().get("SELECT state_json FROM app_state WHERE id = 1");
|
|
771
|
+
return (parseJson(legacy?.state_json, {}).metadataSnapshot ?? null);
|
|
772
|
+
}
|
|
773
|
+
async saveMetadataSnapshot(snapshot) {
|
|
774
|
+
await this.commit([
|
|
775
|
+
{
|
|
776
|
+
sql: `INSERT INTO app_meta (key, value_json) VALUES (?, ?)
|
|
777
|
+
ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json`,
|
|
778
|
+
args: [METADATA_SNAPSHOT_META_KEY, JSON.stringify(snapshot)],
|
|
779
|
+
},
|
|
780
|
+
], ["metadata"]);
|
|
781
|
+
}
|
|
782
|
+
async getFileHistory(path) {
|
|
783
|
+
const pathRow = await this.requireConnection().get("SELECT file_id FROM history_file_paths WHERE path = ?", path);
|
|
784
|
+
if (!pathRow)
|
|
785
|
+
return null;
|
|
786
|
+
const [fileRow, revisionRows] = await Promise.all([
|
|
787
|
+
this.requireConnection().get("SELECT data_json FROM history_files WHERE file_id = ?", pathRow.file_id),
|
|
788
|
+
this.requireConnection().all("SELECT data_json FROM history_revisions WHERE file_id = ? ORDER BY ordinal", pathRow.file_id),
|
|
789
|
+
]);
|
|
790
|
+
if (!fileRow)
|
|
791
|
+
return null;
|
|
792
|
+
return {
|
|
793
|
+
file: parseJson(fileRow.data_json, {
|
|
794
|
+
fileId: pathRow.file_id,
|
|
795
|
+
currentPath: path,
|
|
796
|
+
deleted: false,
|
|
797
|
+
}),
|
|
798
|
+
revisions: revisionRows.map((row) => parseJson(row.data_json, {})),
|
|
799
|
+
};
|
|
800
|
+
}
|
|
801
|
+
async storeFileHistoryRevision(input) {
|
|
802
|
+
const existing = (await this.getFileHistory(input.path)) ??
|
|
803
|
+
(input.previousPath
|
|
804
|
+
? await this.getFileHistory(input.previousPath)
|
|
805
|
+
: null);
|
|
806
|
+
const fileId = existing?.file.fileId ?? createStableId("history-file");
|
|
807
|
+
const latest = existing?.revisions.at(-1);
|
|
808
|
+
const content = input.content ?? latest?.content ?? "";
|
|
809
|
+
const contentHash = input.contentHash ?? latest?.contentHash ?? hashText(content);
|
|
810
|
+
const deduplicate = input.eventType !== "rename" &&
|
|
811
|
+
input.eventType !== "delete" &&
|
|
812
|
+
!existing?.file.deleted &&
|
|
813
|
+
latest?.contentHash === contentHash;
|
|
814
|
+
if (deduplicate)
|
|
815
|
+
return { fileId, stored: false, deduplicated: true };
|
|
816
|
+
const file = {
|
|
817
|
+
fileId,
|
|
818
|
+
currentPath: input.path,
|
|
819
|
+
deleted: input.eventType === "delete",
|
|
820
|
+
};
|
|
821
|
+
const revision = input.replaceLatest && latest
|
|
822
|
+
? {
|
|
823
|
+
...latest,
|
|
824
|
+
currentPath: input.path,
|
|
825
|
+
capturedPath: input.eventType === "rename" && input.previousPath
|
|
826
|
+
? input.previousPath
|
|
827
|
+
: input.path,
|
|
828
|
+
eventType: input.eventType,
|
|
829
|
+
createdAt: input.createdAt,
|
|
830
|
+
sourceMtime: input.sourceMtime,
|
|
831
|
+
sourceSize: input.sourceSize,
|
|
832
|
+
contentHash,
|
|
833
|
+
content,
|
|
834
|
+
}
|
|
835
|
+
: {
|
|
836
|
+
revisionId: createStableId("history-revision"),
|
|
837
|
+
fileId,
|
|
838
|
+
currentPath: input.path,
|
|
839
|
+
capturedPath: input.eventType === "rename" && input.previousPath
|
|
840
|
+
? input.previousPath
|
|
841
|
+
: input.path,
|
|
842
|
+
eventType: input.eventType,
|
|
843
|
+
createdAt: input.createdAt,
|
|
844
|
+
sourceMtime: input.sourceMtime,
|
|
845
|
+
sourceSize: input.sourceSize,
|
|
846
|
+
contentHash,
|
|
847
|
+
content,
|
|
848
|
+
};
|
|
849
|
+
const revisions = input.replaceLatest && latest
|
|
850
|
+
? [...(existing?.revisions.slice(0, -1) ?? []), revision]
|
|
851
|
+
: [...(existing?.revisions ?? []), revision];
|
|
852
|
+
const retained = revisions.slice(-Math.max(1, input.maxRevisions ?? revisions.length));
|
|
853
|
+
const statements = [
|
|
854
|
+
{
|
|
855
|
+
sql: `INSERT INTO history_files (file_id, data_json) VALUES (?, ?)
|
|
856
|
+
ON CONFLICT(file_id) DO UPDATE SET data_json = excluded.data_json`,
|
|
857
|
+
args: [fileId, JSON.stringify(file)],
|
|
858
|
+
},
|
|
859
|
+
{
|
|
860
|
+
sql: "DELETE FROM history_revisions WHERE file_id = ?",
|
|
861
|
+
args: [fileId],
|
|
862
|
+
},
|
|
863
|
+
{
|
|
864
|
+
sql: "DELETE FROM history_file_paths WHERE file_id = ?",
|
|
865
|
+
args: [fileId],
|
|
866
|
+
},
|
|
867
|
+
{
|
|
868
|
+
sql: `INSERT INTO history_file_paths (path, file_id) VALUES (?, ?)
|
|
869
|
+
ON CONFLICT(path) DO UPDATE SET file_id = excluded.file_id`,
|
|
870
|
+
args: [input.path, fileId],
|
|
871
|
+
},
|
|
872
|
+
];
|
|
873
|
+
retained.forEach((entry, ordinal) => statements.push({
|
|
874
|
+
sql: "INSERT INTO history_revisions (file_id, ordinal, data_json) VALUES (?, ?, ?)",
|
|
875
|
+
args: [fileId, ordinal, JSON.stringify(entry)],
|
|
876
|
+
}));
|
|
877
|
+
await this.commit(statements, ["history"], [input.path]);
|
|
878
|
+
return { fileId, stored: true, deduplicated: false, revision };
|
|
879
|
+
}
|
|
880
|
+
async listNotifications() {
|
|
881
|
+
const rows = await this.requireConnection().all("SELECT data_json FROM notifications");
|
|
882
|
+
return rows
|
|
883
|
+
.map((row) => parseJson(row.data_json, {}))
|
|
884
|
+
.filter((record) => !record.cleared)
|
|
885
|
+
.sort((left, right) => right.createdAt - left.createdAt);
|
|
886
|
+
}
|
|
887
|
+
async upsertNotification(record) {
|
|
888
|
+
await this.commit([
|
|
889
|
+
{
|
|
890
|
+
sql: `INSERT INTO notifications (id, data_json) VALUES (?, ?)
|
|
891
|
+
ON CONFLICT(id) DO UPDATE SET data_json = excluded.data_json`,
|
|
892
|
+
args: [record.id, JSON.stringify(record)],
|
|
893
|
+
},
|
|
894
|
+
], ["notification"]);
|
|
895
|
+
}
|
|
896
|
+
async markNotificationRead(id) {
|
|
897
|
+
await this.updateNotification(id, (record) => ({
|
|
898
|
+
...record,
|
|
899
|
+
read: true,
|
|
900
|
+
updatedAt: Date.now(),
|
|
901
|
+
}));
|
|
902
|
+
}
|
|
903
|
+
async clearNotification(id) {
|
|
904
|
+
await this.updateNotification(id, (record) => ({
|
|
905
|
+
...record,
|
|
906
|
+
cleared: true,
|
|
907
|
+
updatedAt: Date.now(),
|
|
908
|
+
}));
|
|
909
|
+
}
|
|
910
|
+
async clearAllNotifications() {
|
|
911
|
+
const rows = await this.requireConnection().all("SELECT id, data_json FROM notifications");
|
|
912
|
+
const now = Date.now();
|
|
913
|
+
await this.commit(rows.map((row) => ({
|
|
914
|
+
sql: "UPDATE notifications SET data_json = ? WHERE id = ?",
|
|
915
|
+
args: [
|
|
916
|
+
JSON.stringify({
|
|
917
|
+
...parseJson(row.data_json, {}),
|
|
918
|
+
cleared: true,
|
|
919
|
+
updatedAt: now,
|
|
920
|
+
}),
|
|
921
|
+
row.id,
|
|
922
|
+
],
|
|
923
|
+
})), ["notification"]);
|
|
924
|
+
}
|
|
925
|
+
async getMemorySourceState(sourceKey) {
|
|
926
|
+
const row = await this.requireConnection().get("SELECT data_json FROM memory_source_state WHERE source_key = ?", sourceKey);
|
|
927
|
+
return row
|
|
928
|
+
? parseJson(row.data_json, {})
|
|
929
|
+
: undefined;
|
|
930
|
+
}
|
|
931
|
+
async listMemorySourceStates() {
|
|
932
|
+
const rows = await this.requireConnection().all("SELECT data_json FROM memory_source_state ORDER BY source_key");
|
|
933
|
+
return rows.map((row) => parseJson(row.data_json, {}));
|
|
934
|
+
}
|
|
935
|
+
async upsertMemorySourceState(record) {
|
|
936
|
+
await this.commit([
|
|
937
|
+
{
|
|
938
|
+
sql: `INSERT INTO memory_source_state
|
|
939
|
+
(source_key, source_path, source_hash, status, indexed_at, data_json)
|
|
940
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
941
|
+
ON CONFLICT(source_key) DO UPDATE SET
|
|
942
|
+
source_path = excluded.source_path,
|
|
943
|
+
source_hash = excluded.source_hash,
|
|
944
|
+
status = excluded.status,
|
|
945
|
+
indexed_at = excluded.indexed_at,
|
|
946
|
+
data_json = excluded.data_json`,
|
|
947
|
+
args: [
|
|
948
|
+
record.sourceKey,
|
|
949
|
+
record.sourcePath,
|
|
950
|
+
record.sourceHash,
|
|
951
|
+
record.status,
|
|
952
|
+
record.indexedAt,
|
|
953
|
+
JSON.stringify(record),
|
|
954
|
+
],
|
|
955
|
+
},
|
|
956
|
+
], ["memory"], [record.sourcePath]);
|
|
957
|
+
}
|
|
958
|
+
async deleteMemorySourceState(sourceKey) {
|
|
959
|
+
const existing = await this.getMemorySourceState(sourceKey);
|
|
960
|
+
await this.commit([
|
|
961
|
+
{
|
|
962
|
+
sql: "DELETE FROM memory_source_state WHERE source_key = ?",
|
|
963
|
+
args: [sourceKey],
|
|
964
|
+
},
|
|
965
|
+
], ["memory"], existing ? [existing.sourcePath] : []);
|
|
966
|
+
}
|
|
967
|
+
async upsertMemoryCandidate(input) {
|
|
968
|
+
const candidate = input.candidate;
|
|
969
|
+
await this.commit([
|
|
970
|
+
{
|
|
971
|
+
sql: `INSERT INTO memory_candidates
|
|
972
|
+
(id, scope_kind, scope_path, state, last_seen_at, data_json)
|
|
973
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
974
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
975
|
+
scope_kind = excluded.scope_kind,
|
|
976
|
+
scope_path = excluded.scope_path,
|
|
977
|
+
state = excluded.state,
|
|
978
|
+
last_seen_at = excluded.last_seen_at,
|
|
979
|
+
data_json = excluded.data_json`,
|
|
980
|
+
args: [
|
|
981
|
+
candidate.id,
|
|
982
|
+
candidate.scopeKind,
|
|
983
|
+
candidate.scopePath,
|
|
984
|
+
candidate.state,
|
|
985
|
+
candidate.lastSeenAt,
|
|
986
|
+
JSON.stringify(candidate),
|
|
987
|
+
],
|
|
988
|
+
},
|
|
989
|
+
{
|
|
990
|
+
sql: "DELETE FROM memory_candidate_origins WHERE candidate_id = ?",
|
|
991
|
+
args: [candidate.id],
|
|
992
|
+
},
|
|
993
|
+
...input.origins
|
|
994
|
+
.filter((origin) => origin.candidateId === candidate.id)
|
|
995
|
+
.map((origin) => ({
|
|
996
|
+
sql: `INSERT INTO memory_candidate_origins
|
|
997
|
+
(candidate_id, conversation_id, entry_id, data_json)
|
|
998
|
+
VALUES (?, ?, ?, ?)`,
|
|
999
|
+
args: [
|
|
1000
|
+
origin.candidateId,
|
|
1001
|
+
origin.conversationId,
|
|
1002
|
+
origin.entryId,
|
|
1003
|
+
JSON.stringify(origin),
|
|
1004
|
+
],
|
|
1005
|
+
})),
|
|
1006
|
+
], ["memory"]);
|
|
1007
|
+
}
|
|
1008
|
+
async queryMemoryCandidates(query = {}) {
|
|
1009
|
+
const clauses = [];
|
|
1010
|
+
const args = [];
|
|
1011
|
+
if (query.scopeKind) {
|
|
1012
|
+
clauses.push("scope_kind = ?");
|
|
1013
|
+
args.push(query.scopeKind);
|
|
1014
|
+
}
|
|
1015
|
+
if (query.scopePath !== undefined) {
|
|
1016
|
+
clauses.push("scope_path = ?");
|
|
1017
|
+
args.push(query.scopePath);
|
|
1018
|
+
}
|
|
1019
|
+
if (query.states?.length) {
|
|
1020
|
+
clauses.push(`state IN (${query.states.map(() => "?").join(", ")})`);
|
|
1021
|
+
args.push(...query.states);
|
|
1022
|
+
}
|
|
1023
|
+
const limit = Math.max(1, Math.min(query.limit ?? 1_000, 10_000));
|
|
1024
|
+
const rows = await this.requireConnection().all(`SELECT id, data_json FROM memory_candidates
|
|
1025
|
+
${clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""}
|
|
1026
|
+
ORDER BY last_seen_at DESC, id ASC LIMIT ?`, ...args, limit);
|
|
1027
|
+
if (rows.length === 0)
|
|
1028
|
+
return [];
|
|
1029
|
+
const originRows = await this.requireConnection().all(`SELECT candidate_id, data_json FROM memory_candidate_origins
|
|
1030
|
+
WHERE candidate_id IN (${rows.map(() => "?").join(", ")})
|
|
1031
|
+
ORDER BY candidate_id, conversation_id, entry_id`, ...rows.map((row) => row.id));
|
|
1032
|
+
const origins = new Map();
|
|
1033
|
+
for (const row of originRows) {
|
|
1034
|
+
const values = origins.get(row.candidate_id) ?? [];
|
|
1035
|
+
values.push(parseJson(row.data_json, {}));
|
|
1036
|
+
origins.set(row.candidate_id, values);
|
|
1037
|
+
}
|
|
1038
|
+
return rows.map((row) => ({
|
|
1039
|
+
candidate: parseJson(row.data_json, {}),
|
|
1040
|
+
origins: origins.get(row.id) ?? [],
|
|
1041
|
+
}));
|
|
1042
|
+
}
|
|
1043
|
+
async deleteMemoryCandidatesByConversation(conversationId) {
|
|
1044
|
+
const rows = await this.requireConnection().all(`SELECT DISTINCT candidate_id FROM memory_candidate_origins
|
|
1045
|
+
WHERE conversation_id = ?`, conversationId);
|
|
1046
|
+
if (rows.length === 0)
|
|
1047
|
+
return;
|
|
1048
|
+
const ids = rows.map((row) => row.candidate_id);
|
|
1049
|
+
const placeholders = ids.map(() => "?").join(", ");
|
|
1050
|
+
await this.commit([
|
|
1051
|
+
{
|
|
1052
|
+
sql: `DELETE FROM memory_candidate_origins
|
|
1053
|
+
WHERE candidate_id IN (${placeholders})`,
|
|
1054
|
+
args: ids,
|
|
1055
|
+
},
|
|
1056
|
+
{
|
|
1057
|
+
sql: `DELETE FROM memory_candidates WHERE id IN (${placeholders})`,
|
|
1058
|
+
args: ids,
|
|
1059
|
+
},
|
|
1060
|
+
], ["memory"]);
|
|
1061
|
+
}
|
|
1062
|
+
async recordMemoryRecallSignal(record) {
|
|
1063
|
+
await this.commit([
|
|
1064
|
+
{
|
|
1065
|
+
sql: `INSERT INTO memory_recall_signals
|
|
1066
|
+
(target_ref, query_fingerprint, day, best_score, hit_count, data_json)
|
|
1067
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1068
|
+
ON CONFLICT(target_ref, query_fingerprint, day) DO UPDATE SET
|
|
1069
|
+
best_score = max(best_score, excluded.best_score),
|
|
1070
|
+
hit_count = hit_count + excluded.hit_count,
|
|
1071
|
+
data_json = excluded.data_json`,
|
|
1072
|
+
args: [
|
|
1073
|
+
record.targetRef,
|
|
1074
|
+
record.queryFingerprint,
|
|
1075
|
+
record.day,
|
|
1076
|
+
record.bestScore,
|
|
1077
|
+
Math.max(0, record.hitCount),
|
|
1078
|
+
JSON.stringify(record),
|
|
1079
|
+
],
|
|
1080
|
+
},
|
|
1081
|
+
], ["memory"]);
|
|
1082
|
+
}
|
|
1083
|
+
async listMemoryRecallSignals(targetRef) {
|
|
1084
|
+
const rows = await this.requireConnection().all(`SELECT target_ref, query_fingerprint, day, best_score, hit_count, data_json
|
|
1085
|
+
FROM memory_recall_signals
|
|
1086
|
+
${targetRef === undefined ? "" : "WHERE target_ref = ?"}
|
|
1087
|
+
ORDER BY day DESC, query_fingerprint`, ...(targetRef === undefined ? [] : [targetRef]));
|
|
1088
|
+
return rows.map((row) => ({
|
|
1089
|
+
...parseJson(row.data_json, {
|
|
1090
|
+
targetRef: row.target_ref,
|
|
1091
|
+
queryFingerprint: row.query_fingerprint,
|
|
1092
|
+
day: row.day,
|
|
1093
|
+
bestScore: Number(row.best_score),
|
|
1094
|
+
hitCount: Number(row.hit_count),
|
|
1095
|
+
}),
|
|
1096
|
+
bestScore: Number(row.best_score),
|
|
1097
|
+
hitCount: Number(row.hit_count),
|
|
1098
|
+
}));
|
|
1099
|
+
}
|
|
1100
|
+
async claimMemoryJob(input) {
|
|
1101
|
+
const row = await this.requireConnection().get("SELECT data_json FROM memory_jobs WHERE id = ?", input.job.id);
|
|
1102
|
+
const existing = row
|
|
1103
|
+
? parseJson(row.data_json, {})
|
|
1104
|
+
: undefined;
|
|
1105
|
+
if (existing?.status === "completed")
|
|
1106
|
+
return null;
|
|
1107
|
+
const renewing = existing?.status === "running" && existing.ownerId === input.ownerId;
|
|
1108
|
+
if (existing?.status === "running" &&
|
|
1109
|
+
existing.ownerId !== input.ownerId &&
|
|
1110
|
+
(existing.leaseUntil ?? 0) > input.now) {
|
|
1111
|
+
return null;
|
|
1112
|
+
}
|
|
1113
|
+
const base = existing ?? input.job;
|
|
1114
|
+
const attempts = renewing ? base.attempts : base.attempts + 1;
|
|
1115
|
+
if (attempts > base.maxAttempts)
|
|
1116
|
+
return null;
|
|
1117
|
+
const claimed = {
|
|
1118
|
+
...base,
|
|
1119
|
+
status: "running",
|
|
1120
|
+
attempts,
|
|
1121
|
+
ownerId: input.ownerId,
|
|
1122
|
+
leaseUntil: input.now + Math.max(1, input.leaseMs),
|
|
1123
|
+
startedAt: renewing ? base.startedAt : input.now,
|
|
1124
|
+
finishedAt: undefined,
|
|
1125
|
+
errorCode: undefined,
|
|
1126
|
+
};
|
|
1127
|
+
await this.commit([
|
|
1128
|
+
{
|
|
1129
|
+
sql: `INSERT INTO memory_jobs
|
|
1130
|
+
(id, scope_key, status, lease_until, created_at, data_json)
|
|
1131
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1132
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1133
|
+
scope_key = excluded.scope_key,
|
|
1134
|
+
status = excluded.status,
|
|
1135
|
+
lease_until = excluded.lease_until,
|
|
1136
|
+
created_at = excluded.created_at,
|
|
1137
|
+
data_json = excluded.data_json`,
|
|
1138
|
+
args: [
|
|
1139
|
+
claimed.id,
|
|
1140
|
+
claimed.scopeKey,
|
|
1141
|
+
claimed.status,
|
|
1142
|
+
claimed.leaseUntil ?? null,
|
|
1143
|
+
claimed.createdAt,
|
|
1144
|
+
JSON.stringify(claimed),
|
|
1145
|
+
],
|
|
1146
|
+
},
|
|
1147
|
+
], ["memory"]);
|
|
1148
|
+
return claimed;
|
|
1149
|
+
}
|
|
1150
|
+
async updateMemoryJob(input) {
|
|
1151
|
+
const row = await this.requireConnection().get("SELECT data_json FROM memory_jobs WHERE id = ?", input.jobId);
|
|
1152
|
+
if (!row)
|
|
1153
|
+
return null;
|
|
1154
|
+
const existing = parseJson(row.data_json, {});
|
|
1155
|
+
if (existing.status !== "running" ||
|
|
1156
|
+
existing.ownerId !== input.ownerId ||
|
|
1157
|
+
(existing.leaseUntil ?? 0) < input.now) {
|
|
1158
|
+
return null;
|
|
1159
|
+
}
|
|
1160
|
+
const updated = {
|
|
1161
|
+
...existing,
|
|
1162
|
+
...input.patch,
|
|
1163
|
+
ownerId: input.patch.status === "running" ? existing.ownerId : undefined,
|
|
1164
|
+
leaseUntil: input.patch.status === "running"
|
|
1165
|
+
? input.patch.leaseUntil ?? existing.leaseUntil
|
|
1166
|
+
: undefined,
|
|
1167
|
+
};
|
|
1168
|
+
await this.commit([
|
|
1169
|
+
{
|
|
1170
|
+
sql: `UPDATE memory_jobs
|
|
1171
|
+
SET status = ?, lease_until = ?, data_json = ? WHERE id = ?`,
|
|
1172
|
+
args: [
|
|
1173
|
+
updated.status,
|
|
1174
|
+
updated.leaseUntil ?? null,
|
|
1175
|
+
JSON.stringify(updated),
|
|
1176
|
+
updated.id,
|
|
1177
|
+
],
|
|
1178
|
+
},
|
|
1179
|
+
], ["memory"]);
|
|
1180
|
+
return updated;
|
|
1181
|
+
}
|
|
1182
|
+
async listMemoryJobs(scopeKey) {
|
|
1183
|
+
const rows = await this.requireConnection().all(`SELECT data_json FROM memory_jobs
|
|
1184
|
+
${scopeKey === undefined ? "" : "WHERE scope_key = ?"}
|
|
1185
|
+
ORDER BY created_at DESC, id`, ...(scopeKey === undefined ? [] : [scopeKey]));
|
|
1186
|
+
return rows.map((row) => parseJson(row.data_json, {}));
|
|
1187
|
+
}
|
|
1188
|
+
async clearMemoryDerivedState() {
|
|
1189
|
+
await this.commit([
|
|
1190
|
+
"DELETE FROM memory_candidate_origins",
|
|
1191
|
+
"DELETE FROM memory_candidates",
|
|
1192
|
+
"DELETE FROM memory_source_state",
|
|
1193
|
+
"DELETE FROM memory_recall_signals",
|
|
1194
|
+
"DELETE FROM memory_jobs",
|
|
1195
|
+
], ["memory"]);
|
|
1196
|
+
}
|
|
1197
|
+
async getChangeRevision() {
|
|
1198
|
+
return this.currentRevision;
|
|
1199
|
+
}
|
|
1200
|
+
subscribeToChanges(listener) {
|
|
1201
|
+
this.changeListeners.add(listener);
|
|
1202
|
+
return () => this.changeListeners.delete(listener);
|
|
1203
|
+
}
|
|
1204
|
+
async upsertIndexedFile(record) {
|
|
1205
|
+
await this.commit(indexedFileStatements(record), ["metadata"], [record.file.path]);
|
|
1206
|
+
if (record.task)
|
|
1207
|
+
await this.upsertTaskProjection(record.task);
|
|
1208
|
+
else if (record.task === null)
|
|
1209
|
+
await this.deleteTaskProjection(record.file.path);
|
|
1210
|
+
}
|
|
1211
|
+
async getIndexedFile(path) {
|
|
1212
|
+
return this.readIndexedFile(path, false);
|
|
1213
|
+
}
|
|
1214
|
+
async listIndexedFileManifest(query = {}) {
|
|
1215
|
+
const limit = Math.max(1, query.limit ?? 500);
|
|
1216
|
+
const paths = [...new Set(query.paths ?? [])];
|
|
1217
|
+
const pathFilter = paths.length
|
|
1218
|
+
? `AND f.path IN (${paths.map(() => "?").join(", ")})`
|
|
1219
|
+
: "";
|
|
1220
|
+
const rows = await this.requireConnection().all(`SELECT f.path, f.normalized_path, f.extension, f.mtime, f.size, f.hash,
|
|
1221
|
+
f.indexed, f.deleted, m.parser_version
|
|
1222
|
+
FROM files f
|
|
1223
|
+
LEFT JOIN metadata m ON m.path = f.path
|
|
1224
|
+
WHERE f.indexed = 1 AND f.deleted = 0 AND f.path > ? ${pathFilter}
|
|
1225
|
+
ORDER BY f.path LIMIT ?`, query.after ?? "", ...paths, limit + 1);
|
|
1226
|
+
const page = rows.slice(0, limit).map((row) => this.fileRecordFromRow(row));
|
|
1227
|
+
return {
|
|
1228
|
+
rows: page,
|
|
1229
|
+
nextCursor: rows.length > limit ? page.at(-1)?.path : undefined,
|
|
1230
|
+
};
|
|
1231
|
+
}
|
|
1232
|
+
async queryIndexedMetadata(query = {}) {
|
|
1233
|
+
return (await this.queryIndexedMetadataPage({
|
|
1234
|
+
query,
|
|
1235
|
+
limit: query.limit ?? 10_000,
|
|
1236
|
+
})).rows;
|
|
1237
|
+
}
|
|
1238
|
+
async queryIndexedMetadataPage(input = {}) {
|
|
1239
|
+
const limit = Math.max(1, input.limit ?? input.query?.limit ?? 100);
|
|
1240
|
+
const { sql, args } = this.compileMetadataPathQuery(input.query ?? {}, input.after, limit + 1);
|
|
1241
|
+
const rows = await this.requireConnection().all(sql, ...args);
|
|
1242
|
+
const pagePaths = rows.slice(0, limit).map((row) => row.path);
|
|
1243
|
+
const materialized = await this.readIndexedFiles(pagePaths, input.include);
|
|
1244
|
+
return {
|
|
1245
|
+
rows: materialized,
|
|
1246
|
+
nextCursor: rows.length > limit ? pagePaths.at(-1) : undefined,
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
async queryMetadataFacets(query) {
|
|
1250
|
+
const limit = Math.max(1, query.limit ?? 100);
|
|
1251
|
+
const prefixArgs = [];
|
|
1252
|
+
const prefix = this.compilePathPrefixes("path", query.pathPrefixes, prefixArgs);
|
|
1253
|
+
if (query.kind === "tag") {
|
|
1254
|
+
const rows = await this.requireConnection().all(`SELECT ancestor AS value, count(DISTINCT path) AS count
|
|
1255
|
+
FROM metadata_tag_ancestors
|
|
1256
|
+
WHERE 1 = 1 ${prefix}
|
|
1257
|
+
GROUP BY ancestor ORDER BY count DESC, ancestor LIMIT ?`, ...prefixArgs, limit);
|
|
1258
|
+
return rows.map((row) => ({
|
|
1259
|
+
value: row.value,
|
|
1260
|
+
valueType: "string",
|
|
1261
|
+
count: Number(row.count),
|
|
1262
|
+
}));
|
|
1263
|
+
}
|
|
1264
|
+
if (query.kind === "property-name") {
|
|
1265
|
+
const rows = await this.requireConnection().all(`SELECT min(name) AS value, count(DISTINCT path) AS count,
|
|
1266
|
+
group_concat(DISTINCT COALESCE(declared_type, inferred_type)) AS metadata_types
|
|
1267
|
+
FROM metadata_properties
|
|
1268
|
+
WHERE 1 = 1 ${prefix}
|
|
1269
|
+
GROUP BY normalized_name ORDER BY count DESC, normalized_name LIMIT ?`, ...prefixArgs, limit);
|
|
1270
|
+
return rows.map((row) => ({
|
|
1271
|
+
value: row.value,
|
|
1272
|
+
valueType: "string",
|
|
1273
|
+
count: Number(row.count),
|
|
1274
|
+
metadataTypes: row.metadata_types?.split(",").filter(Boolean),
|
|
1275
|
+
topLevel: true,
|
|
1276
|
+
}));
|
|
1277
|
+
}
|
|
1278
|
+
if (query.kind === "property-path") {
|
|
1279
|
+
const rows = await this.requireConnection().all(`SELECT min(value) AS value, count(DISTINCT path) AS count,
|
|
1280
|
+
group_concat(DISTINCT metadata_type) AS metadata_types,
|
|
1281
|
+
max(top_level) AS top_level
|
|
1282
|
+
FROM (
|
|
1283
|
+
SELECT property_path AS value, normalized_property_path AS normalized_value,
|
|
1284
|
+
value_type AS metadata_type, path,
|
|
1285
|
+
CASE WHEN property_name = normalized_property_path THEN 1 ELSE 0 END AS top_level
|
|
1286
|
+
FROM metadata_property_values
|
|
1287
|
+
UNION ALL
|
|
1288
|
+
SELECT name AS value, normalized_name AS normalized_value,
|
|
1289
|
+
COALESCE(declared_type, inferred_type) AS metadata_type, path, 1 AS top_level
|
|
1290
|
+
FROM metadata_properties
|
|
1291
|
+
) property_paths
|
|
1292
|
+
WHERE 1 = 1 ${prefix}
|
|
1293
|
+
GROUP BY normalized_value ORDER BY count DESC, normalized_value LIMIT ?`, ...prefixArgs, limit);
|
|
1294
|
+
return rows.map((row) => ({
|
|
1295
|
+
value: String(row.value).replace(/\[\d+\]/gu, "[]"),
|
|
1296
|
+
valueType: "string",
|
|
1297
|
+
count: Number(row.count),
|
|
1298
|
+
metadataTypes: String(row.metadata_types ?? "")
|
|
1299
|
+
.split(",")
|
|
1300
|
+
.filter(Boolean),
|
|
1301
|
+
topLevel: Boolean(row.top_level),
|
|
1302
|
+
}));
|
|
1303
|
+
}
|
|
1304
|
+
if (!query.propertyName)
|
|
1305
|
+
return [];
|
|
1306
|
+
const rows = await this.requireConnection().all(`SELECT value_type, text_value, number_value, boolean_value, date_value,
|
|
1307
|
+
count(DISTINCT path) AS count
|
|
1308
|
+
FROM metadata_property_values
|
|
1309
|
+
WHERE (property_name = ? OR normalized_property_path = ?) ${prefix}
|
|
1310
|
+
GROUP BY value_type, text_value, number_value, boolean_value, date_value
|
|
1311
|
+
ORDER BY count DESC LIMIT ?`, normalizePropertyPath(query.propertyName), normalizePropertyPath(query.propertyName), ...prefixArgs, limit);
|
|
1312
|
+
return rows.map((row) => ({
|
|
1313
|
+
value: row.value_type === "number"
|
|
1314
|
+
? Number(row.number_value)
|
|
1315
|
+
: row.value_type === "boolean"
|
|
1316
|
+
? Boolean(row.boolean_value)
|
|
1317
|
+
: row.value_type === "null"
|
|
1318
|
+
? null
|
|
1319
|
+
: String(row.date_value ?? row.text_value ?? ""),
|
|
1320
|
+
valueType: String(row.value_type),
|
|
1321
|
+
count: Number(row.count),
|
|
1322
|
+
}));
|
|
1323
|
+
}
|
|
1324
|
+
async queryMetadataLinks(query) {
|
|
1325
|
+
const paths = [
|
|
1326
|
+
...new Set([...(query.paths ?? []), ...(query.path ? [query.path] : [])]),
|
|
1327
|
+
];
|
|
1328
|
+
if (!paths.length)
|
|
1329
|
+
return [];
|
|
1330
|
+
const field = query.direction === "outgoing" ? "source_path" : "resolved_target_path";
|
|
1331
|
+
const conditions = [`${field} IN (${paths.map(() => "?").join(", ")})`];
|
|
1332
|
+
if (query.resolution === "resolved")
|
|
1333
|
+
conditions.push("resolution_state = 'resolved'");
|
|
1334
|
+
if (query.resolution === "unresolved")
|
|
1335
|
+
conditions.push("resolution_state = 'unresolved'");
|
|
1336
|
+
const rows = await this.requireConnection().all(`SELECT data_json FROM metadata_links WHERE ${conditions.join(" AND ")}
|
|
1337
|
+
ORDER BY source_path, ordinal LIMIT ?`, ...paths, query.limit ?? 1000);
|
|
1338
|
+
return rows.map((row) => parseJson(row.data_json, {}));
|
|
1339
|
+
}
|
|
1340
|
+
async deleteIndexedFile(path) {
|
|
1341
|
+
await this.commit([
|
|
1342
|
+
{
|
|
1343
|
+
sql: "UPDATE files SET indexed = 0, deleted = 1 WHERE path = ?",
|
|
1344
|
+
args: [path],
|
|
1345
|
+
},
|
|
1346
|
+
...this.deletePathStatements(path),
|
|
1347
|
+
], ["metadata", "search", "task", "projection", "notebook"], [path]);
|
|
1348
|
+
}
|
|
1349
|
+
async renameIndexedFile(oldPath, newPath) {
|
|
1350
|
+
const record = await this.readIndexedFile(oldPath, false);
|
|
1351
|
+
if (!record)
|
|
1352
|
+
return;
|
|
1353
|
+
const renamed = {
|
|
1354
|
+
file: { ...record.file, path: newPath, normalizedPath: newPath },
|
|
1355
|
+
metadata: record.metadata
|
|
1356
|
+
? { ...record.metadata, path: newPath }
|
|
1357
|
+
: {
|
|
1358
|
+
path: newPath,
|
|
1359
|
+
hash: record.file.hash,
|
|
1360
|
+
parserVersion: "unknown",
|
|
1361
|
+
metadata: {},
|
|
1362
|
+
},
|
|
1363
|
+
links: record.links.map((link) => ({ ...link, sourcePath: newPath })),
|
|
1364
|
+
tags: record.tags.map((tag) => ({ ...tag, path: newPath })),
|
|
1365
|
+
properties: record.properties.map((property) => ({
|
|
1366
|
+
...property,
|
|
1367
|
+
path: newPath,
|
|
1368
|
+
})),
|
|
1369
|
+
};
|
|
1370
|
+
const search = await this.getSearchDocument(oldPath);
|
|
1371
|
+
const notebook = await this.getNotebookState(oldPath);
|
|
1372
|
+
const task = await this.getTaskRow({ path: oldPath });
|
|
1373
|
+
const incoming = await this.requireConnection().all("SELECT source_path, ordinal, data_json FROM metadata_links WHERE resolved_target_path = ?", oldPath);
|
|
1374
|
+
const statements = [
|
|
1375
|
+
...this.deleteMetadataPathStatements(oldPath),
|
|
1376
|
+
{ sql: "DELETE FROM files WHERE path = ?", args: [oldPath] },
|
|
1377
|
+
...indexedFileStatements(renamed),
|
|
1378
|
+
{
|
|
1379
|
+
sql: "UPDATE index_projection_sources SET source_path = ? WHERE source_path = ?",
|
|
1380
|
+
args: [newPath, oldPath],
|
|
1381
|
+
},
|
|
1382
|
+
{
|
|
1383
|
+
sql: "UPDATE index_projection_rows SET source_path = ? WHERE source_path = ?",
|
|
1384
|
+
args: [newPath, oldPath],
|
|
1385
|
+
},
|
|
1386
|
+
{
|
|
1387
|
+
sql: "UPDATE index_projection_edges SET target_path = ? WHERE target_path = ?",
|
|
1388
|
+
args: [newPath, oldPath],
|
|
1389
|
+
},
|
|
1390
|
+
];
|
|
1391
|
+
if (notebook) {
|
|
1392
|
+
statements.push({
|
|
1393
|
+
sql: "DELETE FROM app_meta WHERE key = ?",
|
|
1394
|
+
args: [notebookStateMetaKey(oldPath)],
|
|
1395
|
+
}, {
|
|
1396
|
+
sql: `INSERT INTO app_meta (key, value_json) VALUES (?, ?)
|
|
1397
|
+
ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json`,
|
|
1398
|
+
args: [
|
|
1399
|
+
notebookStateMetaKey(newPath),
|
|
1400
|
+
JSON.stringify({ ...notebook, sourcePath: newPath }),
|
|
1401
|
+
],
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
if (search)
|
|
1405
|
+
statements.push(...this.searchDocumentStatements({
|
|
1406
|
+
...search,
|
|
1407
|
+
path: newPath,
|
|
1408
|
+
chunks: search.chunks?.map((chunk) => ({
|
|
1409
|
+
...chunk,
|
|
1410
|
+
id: chunk.id === oldPath
|
|
1411
|
+
? newPath
|
|
1412
|
+
: chunk.id.startsWith(`${oldPath}#`)
|
|
1413
|
+
? `${newPath}${chunk.id.slice(oldPath.length)}`
|
|
1414
|
+
: chunk.id,
|
|
1415
|
+
})),
|
|
1416
|
+
}, oldPath));
|
|
1417
|
+
if (task)
|
|
1418
|
+
statements.push({
|
|
1419
|
+
sql: "DELETE FROM task_records WHERE document_path = ?",
|
|
1420
|
+
args: [oldPath],
|
|
1421
|
+
}, {
|
|
1422
|
+
sql: `INSERT INTO task_records (document_path, data_json) VALUES (?, ?)
|
|
1423
|
+
ON CONFLICT(document_path) DO UPDATE SET data_json = excluded.data_json`,
|
|
1424
|
+
args: [newPath, JSON.stringify({ ...task, documentPath: newPath })],
|
|
1425
|
+
});
|
|
1426
|
+
for (const row of incoming) {
|
|
1427
|
+
const link = parseJson(row.data_json, {});
|
|
1428
|
+
const data = JSON.stringify({ ...link, resolvedTargetPath: newPath });
|
|
1429
|
+
statements.push({
|
|
1430
|
+
sql: "UPDATE metadata_links SET resolved_target_path = ?, data_json = ? WHERE source_path = ? AND ordinal = ?",
|
|
1431
|
+
args: [newPath, data, row.source_path, row.ordinal],
|
|
1432
|
+
}, {
|
|
1433
|
+
sql: "UPDATE links SET data_json = ? WHERE source_path = ? AND ordinal = ?",
|
|
1434
|
+
args: [data, row.source_path, row.ordinal],
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
await this.commit(statements, ["metadata", "search", "task", "projection", "notebook"], [oldPath, newPath], [{ oldPath, newPath }]);
|
|
1438
|
+
}
|
|
1439
|
+
async upsertSearchDocument(document) {
|
|
1440
|
+
const prepared = await this.prepareSearchDocument(document);
|
|
1441
|
+
await this.commit(this.searchDocumentStatements(prepared), ["search"], [prepared.path]);
|
|
1442
|
+
}
|
|
1443
|
+
async deleteSearchDocument(path) {
|
|
1444
|
+
await this.commit([
|
|
1445
|
+
{ sql: "DELETE FROM search_chunks WHERE path = ?", args: [path] },
|
|
1446
|
+
{ sql: "DELETE FROM search_docs WHERE path = ?", args: [path] },
|
|
1447
|
+
], ["search"], [path]);
|
|
1448
|
+
}
|
|
1449
|
+
async getSearchDocument(path) {
|
|
1450
|
+
const row = await this.requireConnection().get("SELECT data_json FROM search_docs WHERE path = ?", path);
|
|
1451
|
+
return row
|
|
1452
|
+
? parseJson(row.data_json, undefined)
|
|
1453
|
+
: undefined;
|
|
1454
|
+
}
|
|
1455
|
+
async listSearchDocumentManifest(query = {}) {
|
|
1456
|
+
const limit = Math.max(1, query.limit ?? 500);
|
|
1457
|
+
const rows = await this.requireConnection().all(`SELECT path, checksum, source_provider_id, metadata_hash,
|
|
1458
|
+
provider_version, projection_signature, source_mtime, source_size
|
|
1459
|
+
FROM search_docs WHERE path > ? ORDER BY path LIMIT ?`, query.after ?? "", limit + 1);
|
|
1460
|
+
const page = rows.slice(0, limit);
|
|
1461
|
+
return {
|
|
1462
|
+
rows: page.map((row) => ({
|
|
1463
|
+
path: String(row.path),
|
|
1464
|
+
checksum: String(row.checksum),
|
|
1465
|
+
sourceProviderId: row.source_provider_id == null
|
|
1466
|
+
? undefined
|
|
1467
|
+
: String(row.source_provider_id),
|
|
1468
|
+
metadataHash: row.metadata_hash == null ? undefined : String(row.metadata_hash),
|
|
1469
|
+
providerVersion: row.provider_version == null
|
|
1470
|
+
? undefined
|
|
1471
|
+
: String(row.provider_version),
|
|
1472
|
+
projectionSignature: row.projection_signature == null
|
|
1473
|
+
? undefined
|
|
1474
|
+
: String(row.projection_signature),
|
|
1475
|
+
sourceMtime: row.source_mtime == null ? undefined : Number(row.source_mtime),
|
|
1476
|
+
sourceSize: row.source_size == null ? undefined : Number(row.source_size),
|
|
1477
|
+
})),
|
|
1478
|
+
nextCursor: rows.length > limit ? String(page.at(-1)?.path) : undefined,
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
async listSearchDocuments() {
|
|
1482
|
+
const rows = await this.requireConnection().all("SELECT data_json FROM search_docs ORDER BY path");
|
|
1483
|
+
return rows.map((row) => parseJson(row.data_json, {}));
|
|
1484
|
+
}
|
|
1485
|
+
async rebuildSearchIndex() {
|
|
1486
|
+
const pageSize = 100;
|
|
1487
|
+
let after = "";
|
|
1488
|
+
let rebuilt = false;
|
|
1489
|
+
while (true) {
|
|
1490
|
+
const rows = await this.requireConnection().all(`SELECT path, data_json FROM search_docs
|
|
1491
|
+
WHERE path > ? ORDER BY path LIMIT ?`, after, pageSize + 1);
|
|
1492
|
+
const page = rows.slice(0, pageSize);
|
|
1493
|
+
if (!page.length)
|
|
1494
|
+
break;
|
|
1495
|
+
const statements = [];
|
|
1496
|
+
const paths = [];
|
|
1497
|
+
for (const row of page) {
|
|
1498
|
+
const document = parseJson(row.data_json, {});
|
|
1499
|
+
const prepared = await this.prepareSearchDocument(document);
|
|
1500
|
+
statements.push(...this.searchDocumentStatements(prepared));
|
|
1501
|
+
paths.push(row.path);
|
|
1502
|
+
}
|
|
1503
|
+
await this.commit(statements, ["search"], paths);
|
|
1504
|
+
rebuilt = true;
|
|
1505
|
+
after = page.at(-1)?.path ?? after;
|
|
1506
|
+
if (rows.length <= pageSize)
|
|
1507
|
+
break;
|
|
1508
|
+
}
|
|
1509
|
+
if (!rebuilt)
|
|
1510
|
+
await this.commit([], ["search"]);
|
|
1511
|
+
}
|
|
1512
|
+
async searchDocuments(query, options = {}) {
|
|
1513
|
+
const requestedMode = options.mode ?? "auto";
|
|
1514
|
+
const allPathsRequired = options.caseSensitive ||
|
|
1515
|
+
isStructuredSearchQuery(query) ||
|
|
1516
|
+
Boolean(options.sourceProviderIds?.length) ||
|
|
1517
|
+
Boolean(options.pathPrefix);
|
|
1518
|
+
if (allPathsRequired) {
|
|
1519
|
+
return this.evaluateSearchDocumentsForPaths(query, options, await this.queryAllSearchPaths(options.limit, options));
|
|
1520
|
+
}
|
|
1521
|
+
const candidates = new Set();
|
|
1522
|
+
const includesLexical = requestedMode !== "vector";
|
|
1523
|
+
const includesVector = requestedMode !== "lexical" &&
|
|
1524
|
+
this.nativeVectorSearch &&
|
|
1525
|
+
Boolean(this.searchEmbeddingProvider);
|
|
1526
|
+
if (includesLexical) {
|
|
1527
|
+
for (const path of await this.queryLexicalPaths(query, options.limit)) {
|
|
1528
|
+
candidates.add(path);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
if (includesVector) {
|
|
1532
|
+
const queryVector = await this.safeEmbedQuery(query);
|
|
1533
|
+
if (queryVector) {
|
|
1534
|
+
for (const path of await this.queryVectorPaths(queryVector, options.limit)) {
|
|
1535
|
+
candidates.add(path);
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
if (requestedMode === "vector" && !includesVector)
|
|
1540
|
+
return [];
|
|
1541
|
+
return this.evaluateSearchDocumentsForPaths(query, options, candidates);
|
|
1542
|
+
}
|
|
1543
|
+
async searchDocumentPaths(query, options = {}) {
|
|
1544
|
+
return (await this.searchDocuments(query, options)).map((result) => result.document.path);
|
|
1545
|
+
}
|
|
1546
|
+
async queryAllSearchPaths(limit = 100, options = {}) {
|
|
1547
|
+
const clauses = [];
|
|
1548
|
+
const args = [];
|
|
1549
|
+
if (options.sourceProviderIds?.length) {
|
|
1550
|
+
clauses.push(`source_provider_id IN (${options.sourceProviderIds.map(() => "?").join(", ")})`);
|
|
1551
|
+
args.push(...options.sourceProviderIds);
|
|
1552
|
+
}
|
|
1553
|
+
if (options.pathPrefix) {
|
|
1554
|
+
clauses.push("(path = ? OR instr(path, ?) = 1)");
|
|
1555
|
+
args.push(options.pathPrefix, `${options.pathPrefix}/`);
|
|
1556
|
+
}
|
|
1557
|
+
const rows = await this.requireConnection().all(`SELECT path FROM search_docs
|
|
1558
|
+
${clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""}
|
|
1559
|
+
ORDER BY path LIMIT ?`, ...args, Math.max(limit, 100));
|
|
1560
|
+
return rows.map((row) => row.path);
|
|
1561
|
+
}
|
|
1562
|
+
async queryLexicalPaths(query, limit = 100) {
|
|
1563
|
+
const terms = collectSearchQueryTerms(query);
|
|
1564
|
+
if (!terms.length || !this.nativeFullTextSearch) {
|
|
1565
|
+
return this.queryAllSearchPaths(limit);
|
|
1566
|
+
}
|
|
1567
|
+
const ftsQuery = ftsQueryForTerms(terms);
|
|
1568
|
+
const rows = await this.requireConnection().all(`SELECT path,
|
|
1569
|
+
fts_score(name, path, content, tags, metadata_text, ?) AS score
|
|
1570
|
+
FROM search_docs
|
|
1571
|
+
WHERE fts_match(name, path, content, tags, metadata_text, ?)
|
|
1572
|
+
ORDER BY score ASC
|
|
1573
|
+
LIMIT ?`, ftsQuery, ftsQuery, limit);
|
|
1574
|
+
return rows.map((row) => row.path);
|
|
1575
|
+
}
|
|
1576
|
+
async queryVectorPaths(queryVector, limit = 100) {
|
|
1577
|
+
const rows = await this.requireConnection().all(`SELECT path,
|
|
1578
|
+
vector_distance_cos(embedding, vector32(?)) AS distance
|
|
1579
|
+
FROM search_chunks
|
|
1580
|
+
WHERE embedding IS NOT NULL
|
|
1581
|
+
ORDER BY distance ASC
|
|
1582
|
+
LIMIT ?`, JSON.stringify(queryVector), limit);
|
|
1583
|
+
return [...new Set(rows.map((row) => row.path))];
|
|
1584
|
+
}
|
|
1585
|
+
async upsertTaskProjection(record) {
|
|
1586
|
+
await this.ensureTasksProjectionDefinition();
|
|
1587
|
+
await this.commit([
|
|
1588
|
+
{
|
|
1589
|
+
sql: `INSERT INTO task_records (document_path, data_json) VALUES (?, ?)
|
|
1590
|
+
ON CONFLICT(document_path) DO UPDATE SET data_json = excluded.data_json`,
|
|
1591
|
+
args: [record.documentPath, JSON.stringify(record)],
|
|
1592
|
+
},
|
|
1593
|
+
], ["task"], [record.documentPath]);
|
|
1594
|
+
await this.replaceProjectionSource({
|
|
1595
|
+
projectionId: PUBLIC_TASKS_PROJECTION_ID,
|
|
1596
|
+
sourcePath: record.documentPath,
|
|
1597
|
+
sourceHash: (await this.requireConnection().get("SELECT hash FROM files WHERE path = ?", record.documentPath))?.hash ?? record.documentId,
|
|
1598
|
+
rows: [{ id: record.documentId, kind: record.kind, data: { ...record } }],
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1601
|
+
async deleteTaskProjection(path) {
|
|
1602
|
+
await this.commit([
|
|
1603
|
+
{
|
|
1604
|
+
sql: "DELETE FROM task_records WHERE document_path = ?",
|
|
1605
|
+
args: [path],
|
|
1606
|
+
},
|
|
1607
|
+
...this.deleteProjectionSourceStatements(PUBLIC_TASKS_PROJECTION_ID, path),
|
|
1608
|
+
], ["task", "projection"], [path]);
|
|
1609
|
+
}
|
|
1610
|
+
async queryTasks(query = {}) {
|
|
1611
|
+
return (await this.queryProjection(PUBLIC_TASKS_PROJECTION_ID, taskQueryToIndexQuery(query))).rows;
|
|
1612
|
+
}
|
|
1613
|
+
async getTaskRow(lookup) {
|
|
1614
|
+
if (lookup.id)
|
|
1615
|
+
return ((await this.getProjectionRow(PUBLIC_TASKS_PROJECTION_ID, lookup.id)) ?? undefined);
|
|
1616
|
+
if (!lookup.path)
|
|
1617
|
+
return undefined;
|
|
1618
|
+
const row = await this.requireConnection().get("SELECT data_json FROM task_records WHERE document_path = ?", lookup.path);
|
|
1619
|
+
return row
|
|
1620
|
+
? parseJson(row.data_json, undefined)
|
|
1621
|
+
: undefined;
|
|
1622
|
+
}
|
|
1623
|
+
async listChildLinks(query) {
|
|
1624
|
+
const conditions = ["source_path = ?"];
|
|
1625
|
+
const args = [query.sourcePath];
|
|
1626
|
+
if (query.kind) {
|
|
1627
|
+
conditions.push("link_kind = ?");
|
|
1628
|
+
args.push(query.kind);
|
|
1629
|
+
}
|
|
1630
|
+
const rows = await this.requireConnection().all(`SELECT data_json FROM metadata_links WHERE ${conditions.join(" AND ")} ORDER BY ordinal`, ...args);
|
|
1631
|
+
return rows.map((row) => parseJson(row.data_json, {}));
|
|
1632
|
+
}
|
|
1633
|
+
async listTaskDescendants(path) {
|
|
1634
|
+
const source = await this.requireConnection().get("SELECT row_id FROM index_projection_rows WHERE projection_id = ? AND source_path = ? LIMIT 1", PUBLIC_TASKS_PROJECTION_ID, path);
|
|
1635
|
+
if (!source)
|
|
1636
|
+
return [];
|
|
1637
|
+
const seen = new Set([source.row_id]);
|
|
1638
|
+
const queue = [source.row_id];
|
|
1639
|
+
const result = [];
|
|
1640
|
+
while (queue.length) {
|
|
1641
|
+
const sourceRowId = queue.shift();
|
|
1642
|
+
const edges = await this.requireConnection().all(`SELECT target_row_id, target_path FROM index_projection_edges
|
|
1643
|
+
WHERE projection_id = ? AND source_row_id = ? AND relation IN ('task-entry', 'list-entry')
|
|
1644
|
+
ORDER BY ordinal`, PUBLIC_TASKS_PROJECTION_ID, sourceRowId);
|
|
1645
|
+
for (const edge of edges) {
|
|
1646
|
+
const row = edge.target_row_id
|
|
1647
|
+
? await this.getProjectionRow(PUBLIC_TASKS_PROJECTION_ID, edge.target_row_id)
|
|
1648
|
+
: edge.target_path
|
|
1649
|
+
? await this.getTaskRow({ path: edge.target_path })
|
|
1650
|
+
: null;
|
|
1651
|
+
if (!row || seen.has(row.documentId))
|
|
1652
|
+
continue;
|
|
1653
|
+
seen.add(row.documentId);
|
|
1654
|
+
queue.push(row.documentId);
|
|
1655
|
+
result.push(row);
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
return result;
|
|
1659
|
+
}
|
|
1660
|
+
async registerProjectionDefinition(definition) {
|
|
1661
|
+
await this.commit([
|
|
1662
|
+
{
|
|
1663
|
+
sql: `INSERT INTO index_projections
|
|
1664
|
+
(projection_id, owner_plugin_id, schema_version, config_hash, visibility, fields_json, active, updated_at)
|
|
1665
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1666
|
+
ON CONFLICT(projection_id) DO UPDATE SET
|
|
1667
|
+
owner_plugin_id = excluded.owner_plugin_id,
|
|
1668
|
+
schema_version = excluded.schema_version,
|
|
1669
|
+
config_hash = excluded.config_hash,
|
|
1670
|
+
visibility = excluded.visibility,
|
|
1671
|
+
fields_json = excluded.fields_json,
|
|
1672
|
+
active = excluded.active,
|
|
1673
|
+
updated_at = excluded.updated_at`,
|
|
1674
|
+
args: [
|
|
1675
|
+
definition.projectionId,
|
|
1676
|
+
definition.ownerPluginId,
|
|
1677
|
+
definition.schemaVersion,
|
|
1678
|
+
definition.configHash,
|
|
1679
|
+
definition.visibility,
|
|
1680
|
+
JSON.stringify(definition.fields),
|
|
1681
|
+
definition.active ? 1 : 0,
|
|
1682
|
+
definition.updatedAt,
|
|
1683
|
+
],
|
|
1684
|
+
},
|
|
1685
|
+
], ["projection"]);
|
|
1686
|
+
}
|
|
1687
|
+
async unregisterProjectionDefinition(projectionId) {
|
|
1688
|
+
await this.commit([
|
|
1689
|
+
{
|
|
1690
|
+
sql: "UPDATE index_projections SET active = 0, updated_at = ? WHERE projection_id = ?",
|
|
1691
|
+
args: [Date.now(), projectionId],
|
|
1692
|
+
},
|
|
1693
|
+
{
|
|
1694
|
+
sql: "DELETE FROM index_projection_edges WHERE projection_id = ?",
|
|
1695
|
+
args: [projectionId],
|
|
1696
|
+
},
|
|
1697
|
+
{
|
|
1698
|
+
sql: "DELETE FROM index_projection_values WHERE projection_id = ?",
|
|
1699
|
+
args: [projectionId],
|
|
1700
|
+
},
|
|
1701
|
+
{
|
|
1702
|
+
sql: "DELETE FROM index_projection_rows WHERE projection_id = ?",
|
|
1703
|
+
args: [projectionId],
|
|
1704
|
+
},
|
|
1705
|
+
{
|
|
1706
|
+
sql: "DELETE FROM index_projection_sources WHERE projection_id = ?",
|
|
1707
|
+
args: [projectionId],
|
|
1708
|
+
},
|
|
1709
|
+
], ["projection"]);
|
|
1710
|
+
}
|
|
1711
|
+
async replaceProjectionSource(input) {
|
|
1712
|
+
assertProjectionWriteAccess(input.projectionId, input.writerPluginId);
|
|
1713
|
+
if (input.rows.length > MAX_PROJECTION_ROWS_PER_SOURCE)
|
|
1714
|
+
throw new Error(`Projection ${input.projectionId} exceeded ${MAX_PROJECTION_ROWS_PER_SOURCE} rows per source.`);
|
|
1715
|
+
const definition = await this.getProjectionDefinition(input.projectionId);
|
|
1716
|
+
if (!definition?.active)
|
|
1717
|
+
throw new Error("Projection is not registered.");
|
|
1718
|
+
const statements = this.deleteProjectionSourceStatements(input.projectionId, input.sourcePath);
|
|
1719
|
+
statements.push({
|
|
1720
|
+
sql: `INSERT INTO index_projection_sources
|
|
1721
|
+
(projection_id, source_path, source_hash, schema_version, config_hash, status, error, indexed_at)
|
|
1722
|
+
VALUES (?, ?, ?, ?, ?, 'ready', NULL, ?)`,
|
|
1723
|
+
args: [
|
|
1724
|
+
input.projectionId,
|
|
1725
|
+
input.sourcePath,
|
|
1726
|
+
input.sourceHash,
|
|
1727
|
+
definition.schemaVersion,
|
|
1728
|
+
definition.configHash,
|
|
1729
|
+
Date.now(),
|
|
1730
|
+
],
|
|
1731
|
+
});
|
|
1732
|
+
for (const row of input.rows) {
|
|
1733
|
+
statements.push({
|
|
1734
|
+
sql: `INSERT INTO index_projection_rows
|
|
1735
|
+
(projection_id, row_id, source_path, kind, ordinal, data_json)
|
|
1736
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
1737
|
+
args: [
|
|
1738
|
+
input.projectionId,
|
|
1739
|
+
row.id,
|
|
1740
|
+
input.sourcePath,
|
|
1741
|
+
row.kind,
|
|
1742
|
+
row.ordinal ?? 0,
|
|
1743
|
+
JSON.stringify(row.data),
|
|
1744
|
+
],
|
|
1745
|
+
});
|
|
1746
|
+
for (const value of indexedValuesForRow(input.projectionId, row, definition.fields)) {
|
|
1747
|
+
statements.push({
|
|
1748
|
+
sql: `INSERT INTO index_projection_values
|
|
1749
|
+
(projection_id, row_id, field, ordinal, value_type, text_value, number_value, boolean_value, date_value, datetime_value)
|
|
1750
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
1751
|
+
args: [
|
|
1752
|
+
value.projectionId,
|
|
1753
|
+
value.rowId,
|
|
1754
|
+
value.field,
|
|
1755
|
+
value.ordinal,
|
|
1756
|
+
value.valueType,
|
|
1757
|
+
value.textValue ?? null,
|
|
1758
|
+
value.numberValue ?? null,
|
|
1759
|
+
value.booleanValue == null ? null : value.booleanValue ? 1 : 0,
|
|
1760
|
+
value.dateValue ?? null,
|
|
1761
|
+
value.datetimeValue ?? null,
|
|
1762
|
+
],
|
|
1763
|
+
});
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
for (const edge of input.edges ?? [])
|
|
1767
|
+
statements.push({
|
|
1768
|
+
sql: `INSERT INTO index_projection_edges
|
|
1769
|
+
(projection_id, source_row_id, relation, target_projection_id, target_row_id, target_path, target_text, ordinal, data_json)
|
|
1770
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
1771
|
+
args: [
|
|
1772
|
+
input.projectionId,
|
|
1773
|
+
edge.sourceRowId,
|
|
1774
|
+
edge.relation,
|
|
1775
|
+
edge.targetProjectionId ?? null,
|
|
1776
|
+
edge.targetRowId ?? null,
|
|
1777
|
+
edge.targetPath ?? null,
|
|
1778
|
+
edge.targetText ?? null,
|
|
1779
|
+
edge.ordinal,
|
|
1780
|
+
JSON.stringify(edge.data ?? null),
|
|
1781
|
+
],
|
|
1782
|
+
});
|
|
1783
|
+
await this.commit(statements, ["projection"], [input.sourcePath]);
|
|
1784
|
+
}
|
|
1785
|
+
async markProjectionSourceError(input) {
|
|
1786
|
+
assertProjectionWriteAccess(input.projectionId, input.writerPluginId);
|
|
1787
|
+
const definition = await this.getProjectionDefinition(input.projectionId);
|
|
1788
|
+
if (!definition?.active)
|
|
1789
|
+
throw new Error("Projection is not registered.");
|
|
1790
|
+
await this.commit([
|
|
1791
|
+
...this.deleteProjectionSourceStatements(input.projectionId, input.sourcePath),
|
|
1792
|
+
{
|
|
1793
|
+
sql: `INSERT INTO index_projection_sources
|
|
1794
|
+
(projection_id, source_path, source_hash, schema_version, config_hash, status, error, indexed_at)
|
|
1795
|
+
VALUES (?, ?, ?, ?, ?, 'error', ?, ?)`,
|
|
1796
|
+
args: [
|
|
1797
|
+
input.projectionId,
|
|
1798
|
+
input.sourcePath,
|
|
1799
|
+
input.sourceHash,
|
|
1800
|
+
definition.schemaVersion,
|
|
1801
|
+
definition.configHash,
|
|
1802
|
+
input.error,
|
|
1803
|
+
Date.now(),
|
|
1804
|
+
],
|
|
1805
|
+
},
|
|
1806
|
+
], ["projection"], [input.sourcePath]);
|
|
1807
|
+
}
|
|
1808
|
+
async deleteProjectionSource(projectionId, sourcePath, writerPluginId) {
|
|
1809
|
+
assertProjectionWriteAccess(projectionId, writerPluginId);
|
|
1810
|
+
await this.commit(this.deleteProjectionSourceStatements(projectionId, sourcePath), ["projection"], [sourcePath]);
|
|
1811
|
+
}
|
|
1812
|
+
async queryProjection(projectionId, query = {}, readerPluginId) {
|
|
1813
|
+
const definition = await this.getProjectionDefinition(projectionId);
|
|
1814
|
+
assertProjectionReadAccess(definition, readerPluginId);
|
|
1815
|
+
const compiled = compileProjectionQuerySql(projectionId, {
|
|
1816
|
+
...query,
|
|
1817
|
+
after: undefined,
|
|
1818
|
+
limit: query.after ? undefined : query.limit,
|
|
1819
|
+
}, definition);
|
|
1820
|
+
const [rows, sources] = await Promise.all([
|
|
1821
|
+
this.requireConnection().all(compiled.sql, ...compiled.args),
|
|
1822
|
+
this.getProjectionSources(projectionId),
|
|
1823
|
+
]);
|
|
1824
|
+
const records = rows.map((row) => ({
|
|
1825
|
+
projectionId,
|
|
1826
|
+
rowId: String(row.row_id),
|
|
1827
|
+
sourcePath: String(row.source_path),
|
|
1828
|
+
kind: String(row.kind),
|
|
1829
|
+
ordinal: Number(row.ordinal),
|
|
1830
|
+
data: parseJson(row.data_json, {}),
|
|
1831
|
+
}));
|
|
1832
|
+
return evaluateProjectionQuery(records, query, this.currentRevision, projectionIndexStatus(sources));
|
|
1833
|
+
}
|
|
1834
|
+
async getProjectionRow(projectionId, rowId, readerPluginId) {
|
|
1835
|
+
const definition = await this.getProjectionDefinition(projectionId);
|
|
1836
|
+
assertProjectionReadAccess(definition, readerPluginId);
|
|
1837
|
+
const row = await this.requireConnection().get(`SELECT r.data_json FROM index_projection_rows r
|
|
1838
|
+
INNER JOIN index_projection_sources s ON s.projection_id = r.projection_id AND s.source_path = r.source_path
|
|
1839
|
+
LEFT JOIN files f ON f.path = r.source_path
|
|
1840
|
+
WHERE r.projection_id = ?
|
|
1841
|
+
AND (r.row_id = ? OR r.source_path = ? OR json_extract(r.data_json, '$.documentId') = ? OR json_extract(r.data_json, '$.id') = ?)
|
|
1842
|
+
AND s.status = 'ready' AND s.schema_version = ? AND s.config_hash = ?
|
|
1843
|
+
AND (f.path IS NULL OR (f.hash = s.source_hash AND f.deleted = 0))
|
|
1844
|
+
LIMIT 1`, projectionId, rowId, rowId, rowId, rowId, definition.schemaVersion, definition.configHash);
|
|
1845
|
+
return row ? parseJson(row.data_json, null) : null;
|
|
1846
|
+
}
|
|
1847
|
+
async queryRelated(query, readerPluginId) {
|
|
1848
|
+
const definition = await this.getProjectionDefinition(query.projectionId);
|
|
1849
|
+
assertProjectionReadAccess(definition, readerPluginId);
|
|
1850
|
+
const edges = await this.requireConnection().all(query.direction === "in"
|
|
1851
|
+
? `SELECT * FROM index_projection_edges WHERE projection_id = ? AND relation = ? AND target_row_id = ? ORDER BY ordinal LIMIT ?`
|
|
1852
|
+
: `SELECT * FROM index_projection_edges WHERE projection_id = ? AND relation = ? AND source_row_id = ? ORDER BY ordinal LIMIT ?`, query.projectionId, query.relation, query.rowId, query.limit ?? 1000);
|
|
1853
|
+
const rows = [];
|
|
1854
|
+
for (const edge of edges) {
|
|
1855
|
+
const targetId = query.direction === "in"
|
|
1856
|
+
? String(edge.source_row_id)
|
|
1857
|
+
: edge.target_row_id
|
|
1858
|
+
? String(edge.target_row_id)
|
|
1859
|
+
: null;
|
|
1860
|
+
const targetProjection = query.direction === "in"
|
|
1861
|
+
? query.projectionId
|
|
1862
|
+
: String(edge.target_projection_id ?? query.projectionId);
|
|
1863
|
+
const data = targetId
|
|
1864
|
+
? await this.getProjectionRow(targetProjection, targetId, readerPluginId)
|
|
1865
|
+
: edge.target_path
|
|
1866
|
+
? await this.getProjectionRow(targetProjection, String(edge.target_path), readerPluginId)
|
|
1867
|
+
: null;
|
|
1868
|
+
if (!data)
|
|
1869
|
+
continue;
|
|
1870
|
+
rows.push({
|
|
1871
|
+
projectionId: targetProjection,
|
|
1872
|
+
rowId: String(data.id ??
|
|
1873
|
+
data.documentId ??
|
|
1874
|
+
targetId ??
|
|
1875
|
+
edge.target_path),
|
|
1876
|
+
sourcePath: String(data.documentPath ??
|
|
1877
|
+
edge.target_path ??
|
|
1878
|
+
""),
|
|
1879
|
+
kind: "related",
|
|
1880
|
+
ordinal: Number(edge.ordinal),
|
|
1881
|
+
data,
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
return evaluateProjectionQuery(rows, { where: query.targetWhere, limit: query.limit }, this.currentRevision, "ready");
|
|
1885
|
+
}
|
|
1886
|
+
async commit(statements, domains, paths = [], renamed) {
|
|
1887
|
+
const committed = this.commitQueue.then(() => this.commitImmediately(statements, domains, paths, renamed));
|
|
1888
|
+
this.commitQueue = committed.then(() => undefined, () => undefined);
|
|
1889
|
+
return committed;
|
|
1890
|
+
}
|
|
1891
|
+
async commitImmediately(statements, domains, paths = [], renamed) {
|
|
1892
|
+
const revision = this.currentRevision + 1;
|
|
1893
|
+
const change = {
|
|
1894
|
+
revision,
|
|
1895
|
+
domains: [...new Set(domains)],
|
|
1896
|
+
paths: [...new Set(paths)],
|
|
1897
|
+
renamed: renamed?.map((entry) => ({ ...entry })),
|
|
1898
|
+
committedAt: Date.now(),
|
|
1899
|
+
};
|
|
1900
|
+
await this.requireConnection().batch([
|
|
1901
|
+
...statements,
|
|
1902
|
+
{
|
|
1903
|
+
sql: `INSERT INTO app_changes
|
|
1904
|
+
(revision, domains_json, paths_json, renamed_json, committed_at)
|
|
1905
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
1906
|
+
args: [
|
|
1907
|
+
revision,
|
|
1908
|
+
JSON.stringify(change.domains),
|
|
1909
|
+
JSON.stringify(change.paths),
|
|
1910
|
+
JSON.stringify(change.renamed ?? null),
|
|
1911
|
+
change.committedAt,
|
|
1912
|
+
],
|
|
1913
|
+
},
|
|
1914
|
+
{
|
|
1915
|
+
sql: `INSERT INTO app_revision (id, revision) VALUES (1, ?)
|
|
1916
|
+
ON CONFLICT(id) DO UPDATE SET revision = excluded.revision`,
|
|
1917
|
+
args: [revision],
|
|
1918
|
+
},
|
|
1919
|
+
], "immediate");
|
|
1920
|
+
this.currentRevision = revision;
|
|
1921
|
+
for (const listener of this.changeListeners)
|
|
1922
|
+
listener(clone(change));
|
|
1923
|
+
return change;
|
|
1924
|
+
}
|
|
1925
|
+
async updateNotification(id, update) {
|
|
1926
|
+
const row = await this.requireConnection().get("SELECT data_json FROM notifications WHERE id = ?", id);
|
|
1927
|
+
if (!row)
|
|
1928
|
+
return;
|
|
1929
|
+
const record = update(parseJson(row.data_json, {}));
|
|
1930
|
+
await this.commit([
|
|
1931
|
+
{
|
|
1932
|
+
sql: "UPDATE notifications SET data_json = ? WHERE id = ?",
|
|
1933
|
+
args: [JSON.stringify(record), id],
|
|
1934
|
+
},
|
|
1935
|
+
], ["notification"]);
|
|
1936
|
+
}
|
|
1937
|
+
fileRecordFromRow(row) {
|
|
1938
|
+
return {
|
|
1939
|
+
path: String(row.path),
|
|
1940
|
+
normalizedPath: String(row.normalized_path),
|
|
1941
|
+
extension: String(row.extension),
|
|
1942
|
+
mtime: Number(row.mtime),
|
|
1943
|
+
size: Number(row.size),
|
|
1944
|
+
hash: String(row.hash),
|
|
1945
|
+
parserVersion: row.parser_version == null ? undefined : String(row.parser_version),
|
|
1946
|
+
indexed: Boolean(row.indexed),
|
|
1947
|
+
deleted: Boolean(row.deleted),
|
|
1948
|
+
};
|
|
1949
|
+
}
|
|
1950
|
+
async readIndexedFiles(paths, include) {
|
|
1951
|
+
if (!paths.length)
|
|
1952
|
+
return [];
|
|
1953
|
+
const selected = include === undefined ? null : new Set(include);
|
|
1954
|
+
const includes = (domain) => selected === null || selected.has(domain);
|
|
1955
|
+
const chunks = chunkPaths([...new Set(paths)]);
|
|
1956
|
+
const connection = this.requireConnection();
|
|
1957
|
+
const readFiles = async () => {
|
|
1958
|
+
const rows = [];
|
|
1959
|
+
for (const chunk of chunks) {
|
|
1960
|
+
rows.push(...(await connection.all(`SELECT f.path, f.normalized_path, f.extension, f.mtime, f.size,
|
|
1961
|
+
f.hash, f.indexed, f.deleted, m.parser_version
|
|
1962
|
+
FROM files f
|
|
1963
|
+
LEFT JOIN metadata m ON m.path = f.path
|
|
1964
|
+
WHERE f.indexed = 1 AND f.deleted = 0
|
|
1965
|
+
AND f.path IN (${chunk.map(() => "?").join(", ")})
|
|
1966
|
+
ORDER BY f.path`, ...chunk)));
|
|
1967
|
+
}
|
|
1968
|
+
return rows;
|
|
1969
|
+
};
|
|
1970
|
+
const readMetadata = async () => {
|
|
1971
|
+
if (!includes("metadata"))
|
|
1972
|
+
return [];
|
|
1973
|
+
const rows = [];
|
|
1974
|
+
for (const chunk of chunks) {
|
|
1975
|
+
rows.push(...(await connection.all(`SELECT path, hash, parser_version, data_json
|
|
1976
|
+
FROM metadata
|
|
1977
|
+
WHERE path IN (${chunk.map(() => "?").join(", ")})`, ...chunk)));
|
|
1978
|
+
}
|
|
1979
|
+
return rows;
|
|
1980
|
+
};
|
|
1981
|
+
const readDomain = async (domain, table, pathColumn) => {
|
|
1982
|
+
if (!includes(domain))
|
|
1983
|
+
return [];
|
|
1984
|
+
const rows = [];
|
|
1985
|
+
for (const chunk of chunks) {
|
|
1986
|
+
rows.push(...(await connection.all(`SELECT ${pathColumn} AS path, data_json
|
|
1987
|
+
FROM ${table}
|
|
1988
|
+
WHERE ${pathColumn} IN (${chunk.map(() => "?").join(", ")})
|
|
1989
|
+
ORDER BY ${pathColumn}, ordinal`, ...chunk)));
|
|
1990
|
+
}
|
|
1991
|
+
return rows;
|
|
1992
|
+
};
|
|
1993
|
+
const [fileRows, metadataRows, propertyRows, tagRows, linkRows] = await Promise.all([
|
|
1994
|
+
readFiles(),
|
|
1995
|
+
readMetadata(),
|
|
1996
|
+
readDomain("properties", "metadata_properties", "path"),
|
|
1997
|
+
readDomain("tags", "metadata_tags", "path"),
|
|
1998
|
+
readDomain("links", "metadata_links", "source_path"),
|
|
1999
|
+
]);
|
|
2000
|
+
const metadataByPath = new Map(metadataRows.map((row) => [
|
|
2001
|
+
String(row.path),
|
|
2002
|
+
{
|
|
2003
|
+
path: String(row.path),
|
|
2004
|
+
hash: String(row.hash),
|
|
2005
|
+
parserVersion: String(row.parser_version),
|
|
2006
|
+
metadata: parseJson(row.data_json, {}),
|
|
2007
|
+
},
|
|
2008
|
+
]));
|
|
2009
|
+
const collectDomain = (rows) => {
|
|
2010
|
+
const values = new Map();
|
|
2011
|
+
for (const row of rows) {
|
|
2012
|
+
const current = values.get(String(row.path)) ?? [];
|
|
2013
|
+
current.push(parseJson(row.data_json, {}));
|
|
2014
|
+
values.set(String(row.path), current);
|
|
2015
|
+
}
|
|
2016
|
+
return values;
|
|
2017
|
+
};
|
|
2018
|
+
const propertiesByPath = collectDomain(propertyRows);
|
|
2019
|
+
const tagsByPath = collectDomain(tagRows);
|
|
2020
|
+
const linksByPath = collectDomain(linkRows);
|
|
2021
|
+
const filesByPath = new Map(fileRows.map((row) => [String(row.path), this.fileRecordFromRow(row)]));
|
|
2022
|
+
return paths.flatMap((path) => {
|
|
2023
|
+
const file = filesByPath.get(path);
|
|
2024
|
+
if (!file)
|
|
2025
|
+
return [];
|
|
2026
|
+
return [
|
|
2027
|
+
{
|
|
2028
|
+
file,
|
|
2029
|
+
metadata: includes("metadata")
|
|
2030
|
+
? (metadataByPath.get(path) ?? null)
|
|
2031
|
+
: null,
|
|
2032
|
+
properties: includes("properties")
|
|
2033
|
+
? (propertiesByPath.get(path) ?? [])
|
|
2034
|
+
: [],
|
|
2035
|
+
tags: includes("tags") ? (tagsByPath.get(path) ?? []) : [],
|
|
2036
|
+
links: includes("links") ? (linksByPath.get(path) ?? []) : [],
|
|
2037
|
+
},
|
|
2038
|
+
];
|
|
2039
|
+
});
|
|
2040
|
+
}
|
|
2041
|
+
async readIndexedFile(path, legacyOnly) {
|
|
2042
|
+
const file = await this.requireConnection().get(`SELECT path, normalized_path, extension, mtime, size, hash, indexed, deleted
|
|
2043
|
+
FROM files WHERE path = ? AND indexed = 1 AND deleted = 0`, path);
|
|
2044
|
+
if (!file)
|
|
2045
|
+
return undefined;
|
|
2046
|
+
const [metadata, properties, tags, links] = await Promise.all([
|
|
2047
|
+
this.requireConnection().get("SELECT path, hash, parser_version, data_json FROM metadata WHERE path = ?", path),
|
|
2048
|
+
this.requireConnection().all(legacyOnly
|
|
2049
|
+
? "SELECT data_json FROM properties WHERE path = ? ORDER BY ordinal"
|
|
2050
|
+
: "SELECT data_json FROM metadata_properties WHERE path = ? ORDER BY ordinal", path),
|
|
2051
|
+
this.requireConnection().all(legacyOnly
|
|
2052
|
+
? "SELECT data_json FROM tags WHERE path = ? ORDER BY ordinal"
|
|
2053
|
+
: "SELECT data_json FROM metadata_tags WHERE path = ? ORDER BY ordinal", path),
|
|
2054
|
+
this.requireConnection().all(legacyOnly
|
|
2055
|
+
? "SELECT data_json FROM links WHERE source_path = ? ORDER BY ordinal"
|
|
2056
|
+
: "SELECT data_json FROM metadata_links WHERE source_path = ? ORDER BY ordinal", path),
|
|
2057
|
+
]);
|
|
2058
|
+
return {
|
|
2059
|
+
file: this.fileRecordFromRow(file),
|
|
2060
|
+
metadata: metadata
|
|
2061
|
+
? {
|
|
2062
|
+
path: String(metadata.path),
|
|
2063
|
+
hash: String(metadata.hash),
|
|
2064
|
+
parserVersion: String(metadata.parser_version),
|
|
2065
|
+
metadata: parseJson(metadata.data_json, {}),
|
|
2066
|
+
}
|
|
2067
|
+
: null,
|
|
2068
|
+
properties: properties.map((row) => parseJson(row.data_json, {})),
|
|
2069
|
+
tags: tags.map((row) => parseJson(row.data_json, {})),
|
|
2070
|
+
links: links.map((row) => parseJson(row.data_json, {})),
|
|
2071
|
+
};
|
|
2072
|
+
}
|
|
2073
|
+
compileMetadataPathQuery(query, after, limit) {
|
|
2074
|
+
const args = [];
|
|
2075
|
+
const conditions = ["f.indexed = 1", "f.deleted = 0"];
|
|
2076
|
+
if (after) {
|
|
2077
|
+
conditions.push("f.path > ?");
|
|
2078
|
+
args.push(after);
|
|
2079
|
+
}
|
|
2080
|
+
if (query.extensions?.length) {
|
|
2081
|
+
conditions.push(`f.extension IN (${query.extensions.map(() => "?").join(", ")})`);
|
|
2082
|
+
args.push(...query.extensions.map(normalizeExtension));
|
|
2083
|
+
}
|
|
2084
|
+
if (query.pathPrefixes?.length) {
|
|
2085
|
+
const prefixClauses = query.pathPrefixes.map(() => "(f.path = ? OR f.path LIKE ? ESCAPE '\\')");
|
|
2086
|
+
conditions.push(balancedSqlDisjunction(prefixClauses));
|
|
2087
|
+
for (const raw of query.pathPrefixes) {
|
|
2088
|
+
const prefix = raw.replace(/^\/+|\/+$/g, "");
|
|
2089
|
+
args.push(prefix, `${prefix.replaceAll("%", "\\%").replaceAll("_", "\\_")}/%`);
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
if (query.excludeHiddenPaths) {
|
|
2093
|
+
conditions.push("f.path NOT GLOB '.*' AND f.path NOT GLOB '*/.*'");
|
|
2094
|
+
}
|
|
2095
|
+
for (const tag of query.requiredTags ?? []) {
|
|
2096
|
+
conditions.push("EXISTS (SELECT 1 FROM metadata_tag_ancestors ta WHERE ta.path = f.path AND ta.ancestor = ?)");
|
|
2097
|
+
args.push(normalizeTag(tag));
|
|
2098
|
+
}
|
|
2099
|
+
for (const target of query.resolvedTargetPaths ?? []) {
|
|
2100
|
+
conditions.push("EXISTS (SELECT 1 FROM metadata_links ml WHERE ml.source_path = f.path AND ml.resolved_target_path = ?)");
|
|
2101
|
+
args.push(target);
|
|
2102
|
+
}
|
|
2103
|
+
for (const filter of query.propertyFilters ?? []) {
|
|
2104
|
+
conditions.push(this.compilePropertyFilter(filter, args));
|
|
2105
|
+
}
|
|
2106
|
+
const order = (query.sort ?? []).map((sort) => {
|
|
2107
|
+
const direction = sort.direction === "DESC" ? "DESC" : "ASC";
|
|
2108
|
+
if (sort.field.kind === "file")
|
|
2109
|
+
return `f.${sort.field.field === "path" ? "path" : sort.field.field} ${direction}`;
|
|
2110
|
+
args.push(sort.field.name.toLowerCase());
|
|
2111
|
+
return `(SELECT COALESCE(pv.number_value, pv.boolean_value, pv.date_value, pv.text_value)
|
|
2112
|
+
FROM metadata_property_values pv
|
|
2113
|
+
WHERE pv.path = f.path AND pv.property_name = ?
|
|
2114
|
+
ORDER BY pv.property_ordinal, pv.value_ordinal LIMIT 1) ${direction}`;
|
|
2115
|
+
});
|
|
2116
|
+
order.push("f.path ASC");
|
|
2117
|
+
args.push(limit);
|
|
2118
|
+
return {
|
|
2119
|
+
sql: `SELECT f.path FROM files f WHERE ${conditions.join(" AND ")}
|
|
2120
|
+
ORDER BY ${order.join(", ")} LIMIT ?`,
|
|
2121
|
+
args,
|
|
2122
|
+
};
|
|
2123
|
+
}
|
|
2124
|
+
compilePropertyFilter(filter, args) {
|
|
2125
|
+
const name = normalizePropertyPath(filter.name);
|
|
2126
|
+
if (filter.op === "exists" || filter.op === "not-exists") {
|
|
2127
|
+
args.push(name, name);
|
|
2128
|
+
return `${filter.op === "not-exists" ? "NOT " : ""}(
|
|
2129
|
+
EXISTS (
|
|
2130
|
+
SELECT 1 FROM metadata_properties mp
|
|
2131
|
+
WHERE mp.path = f.path AND mp.normalized_name = ?
|
|
2132
|
+
) OR EXISTS (
|
|
2133
|
+
SELECT 1 FROM metadata_property_values pv
|
|
2134
|
+
WHERE pv.path = f.path AND pv.normalized_property_path = ?
|
|
2135
|
+
)
|
|
2136
|
+
)`;
|
|
2137
|
+
}
|
|
2138
|
+
const operator = filter.op === "=" ? "=" : filter.op === "!=" ? "!=" : filter.op;
|
|
2139
|
+
const value = filter.value;
|
|
2140
|
+
const column = typeof value === "number"
|
|
2141
|
+
? "number_value"
|
|
2142
|
+
: typeof value === "boolean"
|
|
2143
|
+
? "boolean_value"
|
|
2144
|
+
: typeof value === "string" && isDateScalar(value)
|
|
2145
|
+
? "date_value"
|
|
2146
|
+
: "text_value";
|
|
2147
|
+
args.push(name, name, typeof value === "boolean" ? (value ? 1 : 0) : (value ?? null));
|
|
2148
|
+
return `EXISTS (
|
|
2149
|
+
SELECT 1 FROM metadata_property_values pv
|
|
2150
|
+
WHERE pv.path = f.path
|
|
2151
|
+
AND (pv.property_name = ? OR pv.normalized_property_path = ?)
|
|
2152
|
+
AND pv.${column} ${operator} ?
|
|
2153
|
+
)`;
|
|
2154
|
+
}
|
|
2155
|
+
compilePathPrefixes(column, prefixes, args) {
|
|
2156
|
+
if (!prefixes?.length)
|
|
2157
|
+
return "";
|
|
2158
|
+
const clauses = prefixes.map(() => `(${column} = ? OR ${column} LIKE ? ESCAPE '\\')`);
|
|
2159
|
+
for (const raw of prefixes) {
|
|
2160
|
+
const prefix = raw.replace(/^\/+|\/+$/g, "");
|
|
2161
|
+
args.push(prefix, `${prefix.replaceAll("%", "\\%").replaceAll("_", "\\_")}/%`);
|
|
2162
|
+
}
|
|
2163
|
+
return `AND ${balancedSqlDisjunction(clauses)}`;
|
|
2164
|
+
}
|
|
2165
|
+
deletePathStatements(path) {
|
|
2166
|
+
return [
|
|
2167
|
+
...this.deleteMetadataPathStatements(path),
|
|
2168
|
+
{ sql: "DELETE FROM search_chunks WHERE path = ?", args: [path] },
|
|
2169
|
+
{ sql: "DELETE FROM search_docs WHERE path = ?", args: [path] },
|
|
2170
|
+
{ sql: "DELETE FROM task_records WHERE document_path = ?", args: [path] },
|
|
2171
|
+
{
|
|
2172
|
+
sql: "DELETE FROM index_projection_edges WHERE source_row_id IN (SELECT row_id FROM index_projection_rows WHERE source_path = ?)",
|
|
2173
|
+
args: [path],
|
|
2174
|
+
},
|
|
2175
|
+
{
|
|
2176
|
+
sql: "DELETE FROM index_projection_values WHERE row_id IN (SELECT row_id FROM index_projection_rows WHERE source_path = ?)",
|
|
2177
|
+
args: [path],
|
|
2178
|
+
},
|
|
2179
|
+
{
|
|
2180
|
+
sql: "DELETE FROM index_projection_rows WHERE source_path = ?",
|
|
2181
|
+
args: [path],
|
|
2182
|
+
},
|
|
2183
|
+
{
|
|
2184
|
+
sql: "DELETE FROM index_projection_sources WHERE source_path = ?",
|
|
2185
|
+
args: [path],
|
|
2186
|
+
},
|
|
2187
|
+
];
|
|
2188
|
+
}
|
|
2189
|
+
deleteMetadataPathStatements(path) {
|
|
2190
|
+
return [
|
|
2191
|
+
{
|
|
2192
|
+
sql: "DELETE FROM app_meta WHERE key = ?",
|
|
2193
|
+
args: [notebookStateMetaKey(path)],
|
|
2194
|
+
},
|
|
2195
|
+
{ sql: "DELETE FROM metadata WHERE path = ?", args: [path] },
|
|
2196
|
+
{ sql: "DELETE FROM links WHERE source_path = ?", args: [path] },
|
|
2197
|
+
{ sql: "DELETE FROM tags WHERE path = ?", args: [path] },
|
|
2198
|
+
{ sql: "DELETE FROM properties WHERE path = ?", args: [path] },
|
|
2199
|
+
{ sql: "DELETE FROM metadata_links WHERE source_path = ?", args: [path] },
|
|
2200
|
+
{ sql: "DELETE FROM metadata_tags WHERE path = ?", args: [path] },
|
|
2201
|
+
{
|
|
2202
|
+
sql: "DELETE FROM metadata_tag_ancestors WHERE path = ?",
|
|
2203
|
+
args: [path],
|
|
2204
|
+
},
|
|
2205
|
+
{ sql: "DELETE FROM metadata_properties WHERE path = ?", args: [path] },
|
|
2206
|
+
{
|
|
2207
|
+
sql: "DELETE FROM metadata_property_values WHERE path = ?",
|
|
2208
|
+
args: [path],
|
|
2209
|
+
},
|
|
2210
|
+
];
|
|
2211
|
+
}
|
|
2212
|
+
searchDocumentStatements(document, previousPath) {
|
|
2213
|
+
const statements = [];
|
|
2214
|
+
if (previousPath) {
|
|
2215
|
+
statements.push({
|
|
2216
|
+
sql: "DELETE FROM search_chunks WHERE path = ?",
|
|
2217
|
+
args: [previousPath],
|
|
2218
|
+
}, { sql: "DELETE FROM search_docs WHERE path = ?", args: [previousPath] });
|
|
2219
|
+
}
|
|
2220
|
+
statements.push({
|
|
2221
|
+
sql: "DELETE FROM search_chunks WHERE path = ?",
|
|
2222
|
+
args: [document.path],
|
|
2223
|
+
}, {
|
|
2224
|
+
sql: `INSERT INTO search_docs
|
|
2225
|
+
(path, source_provider_id, metadata_hash, provider_version,
|
|
2226
|
+
projection_signature, source_mtime, source_size,
|
|
2227
|
+
name, extension, checksum, content, tags,
|
|
2228
|
+
metadata_text, data_json)
|
|
2229
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2230
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
2231
|
+
source_provider_id = excluded.source_provider_id,
|
|
2232
|
+
metadata_hash = excluded.metadata_hash,
|
|
2233
|
+
provider_version = excluded.provider_version,
|
|
2234
|
+
projection_signature = excluded.projection_signature,
|
|
2235
|
+
source_mtime = excluded.source_mtime,
|
|
2236
|
+
source_size = excluded.source_size,
|
|
2237
|
+
name = excluded.name,
|
|
2238
|
+
extension = excluded.extension,
|
|
2239
|
+
checksum = excluded.checksum,
|
|
2240
|
+
content = excluded.content,
|
|
2241
|
+
tags = excluded.tags,
|
|
2242
|
+
metadata_text = excluded.metadata_text,
|
|
2243
|
+
data_json = excluded.data_json`,
|
|
2244
|
+
args: [
|
|
2245
|
+
document.path,
|
|
2246
|
+
document.sourceProviderId ?? null,
|
|
2247
|
+
document.sourceMetadata?.metadataHash ?? null,
|
|
2248
|
+
document.sourceMetadata?.providerVersion ?? null,
|
|
2249
|
+
document.sourceMetadata?.projectionSignature ?? null,
|
|
2250
|
+
document.sourceMetadata?.sourceMtime ?? null,
|
|
2251
|
+
document.sourceMetadata?.sourceSize ?? null,
|
|
2252
|
+
document.name,
|
|
2253
|
+
document.extension,
|
|
2254
|
+
document.checksum,
|
|
2255
|
+
document.content,
|
|
2256
|
+
document.tags.join(" "),
|
|
2257
|
+
document.metadataText ?? "",
|
|
2258
|
+
JSON.stringify(document),
|
|
2259
|
+
],
|
|
2260
|
+
});
|
|
2261
|
+
for (const chunk of document.chunks ?? []) {
|
|
2262
|
+
const vector = chunk.embedding?.vector;
|
|
2263
|
+
statements.push({
|
|
2264
|
+
sql: this.nativeVectorSearch && vector?.length
|
|
2265
|
+
? `INSERT INTO search_chunks (id, path, text, embedding_json, embedding)
|
|
2266
|
+
VALUES (?, ?, ?, ?, vector32(?))`
|
|
2267
|
+
: `INSERT INTO search_chunks (id, path, text, embedding_json, embedding)
|
|
2268
|
+
VALUES (?, ?, ?, ?, NULL)`,
|
|
2269
|
+
args: this.nativeVectorSearch && vector?.length
|
|
2270
|
+
? [
|
|
2271
|
+
chunk.id,
|
|
2272
|
+
document.path,
|
|
2273
|
+
chunk.text,
|
|
2274
|
+
JSON.stringify(chunk.embedding ?? null),
|
|
2275
|
+
JSON.stringify(vector),
|
|
2276
|
+
]
|
|
2277
|
+
: [
|
|
2278
|
+
chunk.id,
|
|
2279
|
+
document.path,
|
|
2280
|
+
chunk.text,
|
|
2281
|
+
JSON.stringify(chunk.embedding ?? null),
|
|
2282
|
+
],
|
|
2283
|
+
});
|
|
2284
|
+
}
|
|
2285
|
+
return statements;
|
|
2286
|
+
}
|
|
2287
|
+
async prepareSearchDocument(document) {
|
|
2288
|
+
const normalized = normalizeSearchDocument(document);
|
|
2289
|
+
const provider = this.searchEmbeddingProvider;
|
|
2290
|
+
if (!provider || !normalized.chunks?.length)
|
|
2291
|
+
return clone(normalized);
|
|
2292
|
+
try {
|
|
2293
|
+
if (!(await provider.ready())) {
|
|
2294
|
+
return {
|
|
2295
|
+
...clone(normalized),
|
|
2296
|
+
chunks: normalized.chunks.map((chunk) => ({
|
|
2297
|
+
...clone(chunk),
|
|
2298
|
+
embedding: {
|
|
2299
|
+
...clone(chunk.embedding),
|
|
2300
|
+
status: "pending",
|
|
2301
|
+
modelId: provider.config.modelId ?? "lapis/token-hash-v0",
|
|
2302
|
+
modelVersion: provider.config.modelVersion,
|
|
2303
|
+
dimensions: provider.config.dimensions,
|
|
2304
|
+
},
|
|
2305
|
+
})),
|
|
2306
|
+
};
|
|
2307
|
+
}
|
|
2308
|
+
const embeddings = new Map((await provider.embedDocument(normalized)).map((entry) => [
|
|
2309
|
+
entry.chunkId,
|
|
2310
|
+
entry,
|
|
2311
|
+
]));
|
|
2312
|
+
const timestamp = Date.now();
|
|
2313
|
+
return {
|
|
2314
|
+
...clone(normalized),
|
|
2315
|
+
chunks: normalized.chunks.map((chunk) => {
|
|
2316
|
+
const embedding = embeddings.get(chunk.id);
|
|
2317
|
+
return embedding
|
|
2318
|
+
? {
|
|
2319
|
+
...clone(chunk),
|
|
2320
|
+
embedding: {
|
|
2321
|
+
status: "ready",
|
|
2322
|
+
modelId: provider.config.modelId ?? "lapis/token-hash-v0",
|
|
2323
|
+
modelVersion: provider.config.modelVersion,
|
|
2324
|
+
dimensions: embedding.vector.length,
|
|
2325
|
+
vector: [...embedding.vector],
|
|
2326
|
+
fingerprint: embedding.fingerprint,
|
|
2327
|
+
dirty: false,
|
|
2328
|
+
updatedAt: timestamp,
|
|
2329
|
+
},
|
|
2330
|
+
}
|
|
2331
|
+
: clone(chunk);
|
|
2332
|
+
}),
|
|
2333
|
+
};
|
|
2334
|
+
}
|
|
2335
|
+
catch (error) {
|
|
2336
|
+
return embeddingErrorState(normalized, provider, error);
|
|
2337
|
+
}
|
|
2338
|
+
}
|
|
2339
|
+
async evaluateSearchDocumentsForPaths(query, options, candidatePaths) {
|
|
2340
|
+
const paths = [...new Set(candidatePaths)];
|
|
2341
|
+
if (!paths.length)
|
|
2342
|
+
return [];
|
|
2343
|
+
const rows = [];
|
|
2344
|
+
for (const chunk of chunkPaths(paths)) {
|
|
2345
|
+
rows.push(...(await this.requireConnection().all(`SELECT path, data_json FROM search_docs
|
|
2346
|
+
WHERE path IN (${chunk.map(() => "?").join(", ")})`, ...chunk)));
|
|
2347
|
+
}
|
|
2348
|
+
const propertyNames = searchPropertyNames(query);
|
|
2349
|
+
const allowedProviders = options.sourceProviderIds?.length
|
|
2350
|
+
? new Set(options.sourceProviderIds)
|
|
2351
|
+
: null;
|
|
2352
|
+
const propertyRows = await this.readIndexedFiles(paths, ["properties"]);
|
|
2353
|
+
const propertiesByPath = new Map(propertyRows.map((row) => [row.file.path, row.properties]));
|
|
2354
|
+
const sourceDocuments = rows.map((row) => {
|
|
2355
|
+
const document = parseJson(row.data_json, {});
|
|
2356
|
+
return {
|
|
2357
|
+
document,
|
|
2358
|
+
properties: searchDocumentProperties(document, propertiesByPath.get(document.path) ?? []),
|
|
2359
|
+
};
|
|
2360
|
+
});
|
|
2361
|
+
const filtered = sourceDocuments.filter(({ document, properties }) => pathWithinPrefix(document.path, options.pathPrefix) &&
|
|
2362
|
+
(!allowedProviders ||
|
|
2363
|
+
(document.sourceProviderId != null &&
|
|
2364
|
+
allowedProviders.has(document.sourceProviderId))) &&
|
|
2365
|
+
hasSearchPropertyNames(properties, propertyNames));
|
|
2366
|
+
const requestedMode = options.mode ?? "auto";
|
|
2367
|
+
const queryVector = requestedMode === "lexical" || !this.searchEmbeddingProvider
|
|
2368
|
+
? null
|
|
2369
|
+
: await this.safeEmbedQuery(query);
|
|
2370
|
+
const vectorScores = new Map(filtered.map(({ document }) => [
|
|
2371
|
+
document.path,
|
|
2372
|
+
queryVector
|
|
2373
|
+
? scoreVectorDocument(document, queryVector)
|
|
2374
|
+
: { score: 0, matchedChunkIds: [] },
|
|
2375
|
+
]));
|
|
2376
|
+
const lexicalScores = new Map(filtered.map(({ document, properties }) => [
|
|
2377
|
+
document.path,
|
|
2378
|
+
scoreSearchDocument(document, query, properties, options),
|
|
2379
|
+
]));
|
|
2380
|
+
const vectorCandidateCount = [...vectorScores.values()].filter((entry) => entry.score >= MIN_VECTOR_SEARCH_SCORE).length;
|
|
2381
|
+
const appliedMode = queryVector
|
|
2382
|
+
? requestedMode === "vector"
|
|
2383
|
+
? "vector"
|
|
2384
|
+
: vectorCandidateCount
|
|
2385
|
+
? "hybrid"
|
|
2386
|
+
: "lexical"
|
|
2387
|
+
: "lexical";
|
|
2388
|
+
const lexicalRanks = rankSearchScores([...lexicalScores].map(([path, score]) => ({ path, score })));
|
|
2389
|
+
const vectorRanks = rankSearchScores([...vectorScores].map(([path, entry]) => ({ path, score: entry.score })), (score) => score >= MIN_VECTOR_SEARCH_SCORE);
|
|
2390
|
+
const results = filtered
|
|
2391
|
+
.map(({ document }) => buildSearchResult(document, query, options, {
|
|
2392
|
+
backendKind: this.kind,
|
|
2393
|
+
appliedMode,
|
|
2394
|
+
lexicalScore: lexicalScores.get(document.path) ?? 0,
|
|
2395
|
+
vectorScore: vectorScores.get(document.path)?.score ?? 0,
|
|
2396
|
+
lexicalCandidateCount: filtered.length,
|
|
2397
|
+
vectorCandidateCount,
|
|
2398
|
+
providerConfig: this.searchEmbeddingProviderConfig,
|
|
2399
|
+
preferredChunkIds: vectorScores.get(document.path)?.matchedChunkIds ?? [],
|
|
2400
|
+
lexicalRank: lexicalRanks.get(document.path),
|
|
2401
|
+
vectorRank: vectorRanks.get(document.path),
|
|
2402
|
+
}))
|
|
2403
|
+
.filter((result) => {
|
|
2404
|
+
const lexicalHit = (result.scoreBreakdown.lexical ?? 0) > 0;
|
|
2405
|
+
const vectorHit = (result.scoreBreakdown.vector ?? 0) >= MIN_VECTOR_SEARCH_SCORE;
|
|
2406
|
+
return appliedMode === "vector"
|
|
2407
|
+
? vectorHit && result.snippets.length > 0
|
|
2408
|
+
: appliedMode === "hybrid"
|
|
2409
|
+
? (lexicalHit || vectorHit) && result.snippets.length > 0
|
|
2410
|
+
: lexicalHit && result.snippets.length > 0;
|
|
2411
|
+
})
|
|
2412
|
+
.sort(compareSearchResults);
|
|
2413
|
+
const diagnostics = resolveSearchQueryEnhancementDiagnostics(results, options);
|
|
2414
|
+
return results.slice(0, options.limit ?? 100).map((result) => {
|
|
2415
|
+
const copy = clone(result);
|
|
2416
|
+
if (copy.diagnostics && diagnostics)
|
|
2417
|
+
copy.diagnostics.queryEnhancement = clone(diagnostics);
|
|
2418
|
+
return copy;
|
|
2419
|
+
});
|
|
2420
|
+
}
|
|
2421
|
+
async safeEmbedQuery(query) {
|
|
2422
|
+
try {
|
|
2423
|
+
return this.searchEmbeddingProvider
|
|
2424
|
+
? await this.searchEmbeddingProvider.embedQuery(query)
|
|
2425
|
+
: null;
|
|
2426
|
+
}
|
|
2427
|
+
catch {
|
|
2428
|
+
return null;
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
deleteProjectionSourceStatements(projectionId, sourcePath) {
|
|
2432
|
+
return [
|
|
2433
|
+
{
|
|
2434
|
+
sql: "DELETE FROM index_projection_edges WHERE projection_id = ? AND source_row_id IN (SELECT row_id FROM index_projection_rows WHERE projection_id = ? AND source_path = ?)",
|
|
2435
|
+
args: [projectionId, projectionId, sourcePath],
|
|
2436
|
+
},
|
|
2437
|
+
{
|
|
2438
|
+
sql: "DELETE FROM index_projection_values WHERE projection_id = ? AND row_id IN (SELECT row_id FROM index_projection_rows WHERE projection_id = ? AND source_path = ?)",
|
|
2439
|
+
args: [projectionId, projectionId, sourcePath],
|
|
2440
|
+
},
|
|
2441
|
+
{
|
|
2442
|
+
sql: "DELETE FROM index_projection_rows WHERE projection_id = ? AND source_path = ?",
|
|
2443
|
+
args: [projectionId, sourcePath],
|
|
2444
|
+
},
|
|
2445
|
+
{
|
|
2446
|
+
sql: "DELETE FROM index_projection_sources WHERE projection_id = ? AND source_path = ?",
|
|
2447
|
+
args: [projectionId, sourcePath],
|
|
2448
|
+
},
|
|
2449
|
+
];
|
|
2450
|
+
}
|
|
2451
|
+
async getProjectionDefinition(projectionId) {
|
|
2452
|
+
const row = await this.requireConnection().get("SELECT * FROM index_projections WHERE projection_id = ?", projectionId);
|
|
2453
|
+
return row
|
|
2454
|
+
? {
|
|
2455
|
+
projectionId: String(row.projection_id),
|
|
2456
|
+
ownerPluginId: String(row.owner_plugin_id),
|
|
2457
|
+
schemaVersion: Number(row.schema_version),
|
|
2458
|
+
configHash: String(row.config_hash),
|
|
2459
|
+
visibility: String(row.visibility),
|
|
2460
|
+
fields: parseJson(row.fields_json, {}),
|
|
2461
|
+
active: Boolean(row.active),
|
|
2462
|
+
updatedAt: Number(row.updated_at),
|
|
2463
|
+
}
|
|
2464
|
+
: undefined;
|
|
2465
|
+
}
|
|
2466
|
+
async getProjectionSources(projectionId) {
|
|
2467
|
+
const rows = await this.requireConnection().all("SELECT * FROM index_projection_sources WHERE projection_id = ?", projectionId);
|
|
2468
|
+
return rows.map((row) => ({
|
|
2469
|
+
projectionId: String(row.projection_id),
|
|
2470
|
+
sourcePath: String(row.source_path),
|
|
2471
|
+
sourceHash: String(row.source_hash),
|
|
2472
|
+
schemaVersion: Number(row.schema_version),
|
|
2473
|
+
configHash: String(row.config_hash),
|
|
2474
|
+
status: String(row.status),
|
|
2475
|
+
error: row.error == null ? null : String(row.error),
|
|
2476
|
+
indexedAt: Number(row.indexed_at),
|
|
2477
|
+
}));
|
|
2478
|
+
}
|
|
2479
|
+
async ensureTasksProjectionDefinition() {
|
|
2480
|
+
if ((await this.getProjectionDefinition(PUBLIC_TASKS_PROJECTION_ID))?.active)
|
|
2481
|
+
return;
|
|
2482
|
+
await this.registerProjectionDefinition({
|
|
2483
|
+
projectionId: PUBLIC_TASKS_PROJECTION_ID,
|
|
2484
|
+
ownerPluginId: "tasks",
|
|
2485
|
+
schemaVersion: TASK_PROJECTION_VERSION,
|
|
2486
|
+
configHash: "",
|
|
2487
|
+
visibility: "public",
|
|
2488
|
+
fields: TASK_PROJECTION_FIELDS,
|
|
2489
|
+
active: true,
|
|
2490
|
+
updatedAt: Date.now(),
|
|
2491
|
+
});
|
|
2492
|
+
}
|
|
2493
|
+
async backfillNormalizedMetadata() {
|
|
2494
|
+
const connection = this.requireConnection();
|
|
2495
|
+
const historyBefore = Number((await connection.get("SELECT count(*) AS count FROM history_revisions"))?.count ?? 0);
|
|
2496
|
+
const paths = await connection.all("SELECT path FROM files WHERE indexed = 1 AND deleted = 0 ORDER BY path");
|
|
2497
|
+
for (const { path } of paths) {
|
|
2498
|
+
const record = await this.readIndexedFile(path, true);
|
|
2499
|
+
if (record?.metadata)
|
|
2500
|
+
await connection.batch(indexedFileStatements({ ...record, metadata: record.metadata }), "immediate");
|
|
2501
|
+
}
|
|
2502
|
+
const historyFiles = await connection.all("SELECT file_id, data_json FROM history_files");
|
|
2503
|
+
const historyPathStatements = historyFiles.flatMap((row) => {
|
|
2504
|
+
const file = parseJson(row.data_json, {
|
|
2505
|
+
fileId: row.file_id,
|
|
2506
|
+
currentPath: "",
|
|
2507
|
+
deleted: false,
|
|
2508
|
+
});
|
|
2509
|
+
return file.currentPath
|
|
2510
|
+
? [
|
|
2511
|
+
{
|
|
2512
|
+
sql: `INSERT INTO history_file_paths (path, file_id) VALUES (?, ?)
|
|
2513
|
+
ON CONFLICT(path) DO UPDATE SET file_id = excluded.file_id`,
|
|
2514
|
+
args: [file.currentPath, row.file_id],
|
|
2515
|
+
},
|
|
2516
|
+
]
|
|
2517
|
+
: [];
|
|
2518
|
+
});
|
|
2519
|
+
if (historyPathStatements.length) {
|
|
2520
|
+
await connection.batch(historyPathStatements, "immediate");
|
|
2521
|
+
}
|
|
2522
|
+
const historyAfter = Number((await connection.get("SELECT count(*) AS count FROM history_revisions"))?.count ?? 0);
|
|
2523
|
+
if (historyBefore !== historyAfter)
|
|
2524
|
+
throw new Error("Turso metadata migration changed History revisions");
|
|
2525
|
+
}
|
|
2526
|
+
async captureMigrationInvariants() {
|
|
2527
|
+
const connection = this.requireConnection();
|
|
2528
|
+
const tables = [
|
|
2529
|
+
"app_state",
|
|
2530
|
+
"files",
|
|
2531
|
+
"metadata",
|
|
2532
|
+
"links",
|
|
2533
|
+
"tags",
|
|
2534
|
+
"properties",
|
|
2535
|
+
"search_docs",
|
|
2536
|
+
"search_chunks",
|
|
2537
|
+
"history_files",
|
|
2538
|
+
"history_revisions",
|
|
2539
|
+
"notifications",
|
|
2540
|
+
"tasks",
|
|
2541
|
+
"task_records",
|
|
2542
|
+
"index_projections",
|
|
2543
|
+
"index_projection_sources",
|
|
2544
|
+
"index_projection_rows",
|
|
2545
|
+
"index_projection_values",
|
|
2546
|
+
"index_projection_edges",
|
|
2547
|
+
];
|
|
2548
|
+
const counts = {};
|
|
2549
|
+
for (const table of tables) {
|
|
2550
|
+
counts[table] = Number((await connection.get(`SELECT count(*) AS count FROM ${table}`))?.count ?? 0);
|
|
2551
|
+
}
|
|
2552
|
+
return counts;
|
|
2553
|
+
}
|
|
2554
|
+
async validateMigration(expectedCounts) {
|
|
2555
|
+
const connection = this.requireConnection();
|
|
2556
|
+
const actualCounts = await this.captureMigrationInvariants();
|
|
2557
|
+
for (const [table, expected] of Object.entries(expectedCounts)) {
|
|
2558
|
+
if (actualCounts[table] !== expected) {
|
|
2559
|
+
throw new Error(`Turso migration changed ${table} row count (${expected} -> ${actualCounts[table]})`);
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
const jsonColumns = [
|
|
2563
|
+
["app_state", "state_json"],
|
|
2564
|
+
["app_meta", "value_json"],
|
|
2565
|
+
["metadata", "data_json"],
|
|
2566
|
+
["links", "data_json"],
|
|
2567
|
+
["tags", "data_json"],
|
|
2568
|
+
["properties", "data_json"],
|
|
2569
|
+
["task_records", "data_json"],
|
|
2570
|
+
["index_projections", "fields_json"],
|
|
2571
|
+
["index_projection_rows", "data_json"],
|
|
2572
|
+
["index_projection_edges", "data_json"],
|
|
2573
|
+
["history_files", "data_json"],
|
|
2574
|
+
["history_revisions", "data_json"],
|
|
2575
|
+
["notifications", "data_json"],
|
|
2576
|
+
["search_docs", "data_json"],
|
|
2577
|
+
["search_chunks", "embedding_json"],
|
|
2578
|
+
];
|
|
2579
|
+
for (const [table, column] of jsonColumns) {
|
|
2580
|
+
const invalid = Number((await connection.get(`SELECT count(*) AS count FROM ${table} WHERE ${column} IS NOT NULL AND json_valid(${column}) = 0`))?.count ?? 0);
|
|
2581
|
+
if (invalid > 0) {
|
|
2582
|
+
throw new Error(`Turso migration validation found ${invalid} invalid ${table}.${column} rows`);
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
for (const [source, normalized] of [
|
|
2586
|
+
["links", "metadata_links"],
|
|
2587
|
+
["tags", "metadata_tags"],
|
|
2588
|
+
["properties", "metadata_properties"],
|
|
2589
|
+
]) {
|
|
2590
|
+
const sourceCount = actualCounts[source] ?? 0;
|
|
2591
|
+
const normalizedCount = Number((await connection.get(`SELECT count(*) AS count FROM ${normalized}`))?.count ?? 0);
|
|
2592
|
+
if (normalizedCount !== sourceCount) {
|
|
2593
|
+
throw new Error(`Turso migration normalized ${normalizedCount}/${sourceCount} ${source} rows`);
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
}
|
|
2597
|
+
async ensureMetadataPropertyPathSchema() {
|
|
2598
|
+
const connection = this.requireConnection();
|
|
2599
|
+
const columns = await connection.all("PRAGMA table_info(metadata_property_values)");
|
|
2600
|
+
if (!columns.some((column) => column.name === "normalized_property_path")) {
|
|
2601
|
+
await connection.exec("ALTER TABLE metadata_property_values ADD COLUMN normalized_property_path TEXT NOT NULL DEFAULT ''");
|
|
2602
|
+
}
|
|
2603
|
+
await connection.exec(`
|
|
2604
|
+
CREATE INDEX IF NOT EXISTS metadata_property_path_text_idx
|
|
2605
|
+
ON metadata_property_values(normalized_property_path, text_value, path);
|
|
2606
|
+
CREATE INDEX IF NOT EXISTS metadata_property_path_number_idx
|
|
2607
|
+
ON metadata_property_values(normalized_property_path, number_value, path);
|
|
2608
|
+
CREATE INDEX IF NOT EXISTS metadata_property_path_boolean_idx
|
|
2609
|
+
ON metadata_property_values(normalized_property_path, boolean_value, path);
|
|
2610
|
+
CREATE INDEX IF NOT EXISTS metadata_property_path_date_idx
|
|
2611
|
+
ON metadata_property_values(normalized_property_path, date_value, path);
|
|
2612
|
+
`);
|
|
2613
|
+
const searchColumns = await connection.all("PRAGMA table_info(search_docs)");
|
|
2614
|
+
const searchColumnNames = new Set(searchColumns.map((column) => column.name));
|
|
2615
|
+
for (const column of [
|
|
2616
|
+
"source_provider_id",
|
|
2617
|
+
"metadata_hash",
|
|
2618
|
+
"provider_version",
|
|
2619
|
+
"projection_signature",
|
|
2620
|
+
"source_mtime",
|
|
2621
|
+
"source_size",
|
|
2622
|
+
]) {
|
|
2623
|
+
if (!searchColumnNames.has(column)) {
|
|
2624
|
+
await connection.exec(`ALTER TABLE search_docs ADD COLUMN ${column} ${column.startsWith("source_") && column !== "source_provider_id" ? "INTEGER" : "TEXT"}`);
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
await connection.exec("CREATE INDEX IF NOT EXISTS search_docs_manifest_idx ON search_docs(source_provider_id, path)");
|
|
2628
|
+
}
|
|
2629
|
+
requireConnection() {
|
|
2630
|
+
if (!this.connection)
|
|
2631
|
+
throw new Error("Turso app database is not open");
|
|
2632
|
+
return this.connection;
|
|
2633
|
+
}
|
|
2634
|
+
}
|
|
2635
|
+
export function canUseTursoWasmDatabase() {
|
|
2636
|
+
return (typeof navigator !== "undefined" &&
|
|
2637
|
+
typeof navigator.storage?.getDirectory === "function" &&
|
|
2638
|
+
typeof SharedArrayBuffer !== "undefined" &&
|
|
2639
|
+
typeof Atomics !== "undefined" &&
|
|
2640
|
+
globalThis.crossOriginIsolated === true);
|
|
2641
|
+
}
|
|
2642
|
+
export async function createTursoWasmConnection(path) {
|
|
2643
|
+
// The Vite-aware export avoids bundling through the package's prebuilt
|
|
2644
|
+
// worker-inline entry, which Rollup's CommonJS resolver can recurse on, and
|
|
2645
|
+
// still keeps the dev-server worker/WASM handling owned by the driver.
|
|
2646
|
+
const { connect } = (await import("@tursodatabase/database-wasm/vite"));
|
|
2647
|
+
return (await connect(path, {
|
|
2648
|
+
experimental: ["index_method"],
|
|
2649
|
+
}));
|
|
2650
|
+
}
|
|
2651
|
+
export class TursoWasmAppDatabaseProvider {
|
|
2652
|
+
id = "turso-wasm-local";
|
|
2653
|
+
canOpen(context) {
|
|
2654
|
+
return context.runtime !== "test" && canUseTursoWasmDatabase();
|
|
2655
|
+
}
|
|
2656
|
+
async open(context) {
|
|
2657
|
+
if (!(await this.canOpen(context))) {
|
|
2658
|
+
throw new Error("Turso WASM requires OPFS and cross-origin isolation");
|
|
2659
|
+
}
|
|
2660
|
+
const database = new TursoAppDatabase(context.vaultId, {
|
|
2661
|
+
kind: "turso-wasm",
|
|
2662
|
+
providerId: this.id,
|
|
2663
|
+
transport: "wasm-worker",
|
|
2664
|
+
role: context.role,
|
|
2665
|
+
crossTabCoordination: context.runtime === "web-pwa",
|
|
2666
|
+
connectionFactory: () => createTursoWasmConnection(tursoWasmDatabasePath(context.vaultId)),
|
|
2667
|
+
});
|
|
2668
|
+
await database.open();
|
|
2669
|
+
return database;
|
|
2670
|
+
}
|
|
2671
|
+
}
|