mindoodb 0.0.24 → 0.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-server.md +9 -1
- package/README.md +5 -3
- package/dist/appendonlystores/network/ClientNetworkContentAddressedStore.d.ts +1 -1
- package/dist/appendonlystores/network/ClientNetworkContentAddressedStore.d.ts.map +1 -1
- package/dist/appendonlystores/network/ClientNetworkContentAddressedStore.js +24 -5
- package/dist/appendonlystores/network/ClientNetworkContentAddressedStore.js.map +1 -1
- package/dist/appendonlystores/network/HttpTransport.d.ts +14 -2
- package/dist/appendonlystores/network/HttpTransport.d.ts.map +1 -1
- package/dist/appendonlystores/network/HttpTransport.js +124 -27
- package/dist/appendonlystores/network/HttpTransport.js.map +1 -1
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.d.ts +159 -4
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.d.ts.map +1 -1
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.js +334 -25
- package/dist/appendonlystores/network/ServerNetworkContentAddressedStore.js.map +1 -1
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.d.ts +9 -0
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.d.ts.map +1 -1
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.js +77 -5
- package/dist/browser/appendonlystores/IndexedDBContentAddressedStore.js.map +1 -1
- package/dist/browser/index.mjs +103 -0
- package/dist/core/BaseMindooDB.d.ts +662 -4
- package/dist/core/BaseMindooDB.d.ts.map +1 -1
- package/dist/core/BaseMindooDB.js +2769 -144
- package/dist/core/BaseMindooDB.js.map +1 -1
- package/dist/core/BaseMindooTenant.d.ts +192 -1
- package/dist/core/BaseMindooTenant.d.ts.map +1 -1
- package/dist/core/BaseMindooTenant.js +633 -88
- package/dist/core/BaseMindooTenant.js.map +1 -1
- package/dist/core/BaseMindooTenantDirectory.d.ts +770 -10
- package/dist/core/BaseMindooTenantDirectory.d.ts.map +1 -1
- package/dist/core/BaseMindooTenantDirectory.js +2715 -292
- package/dist/core/BaseMindooTenantDirectory.js.map +1 -1
- package/dist/core/BaseMindooTenantFactory.d.ts +2 -0
- package/dist/core/BaseMindooTenantFactory.d.ts.map +1 -1
- package/dist/core/BaseMindooTenantFactory.js +20 -0
- package/dist/core/BaseMindooTenantFactory.js.map +1 -1
- package/dist/core/accesscontrol/AccessDeniedError.d.ts +22 -0
- package/dist/core/accesscontrol/AccessDeniedError.d.ts.map +1 -0
- package/dist/core/accesscontrol/AccessDeniedError.js +28 -0
- package/dist/core/accesscontrol/AccessDeniedError.js.map +1 -0
- package/dist/core/accesscontrol/DirectoryStateNode.d.ts +220 -0
- package/dist/core/accesscontrol/DirectoryStateNode.d.ts.map +1 -0
- package/dist/core/accesscontrol/DirectoryStateNode.js +297 -0
- package/dist/core/accesscontrol/DirectoryStateNode.js.map +1 -0
- package/dist/core/accesscontrol/DirectoryTimeTravelIndex.d.ts +118 -0
- package/dist/core/accesscontrol/DirectoryTimeTravelIndex.d.ts.map +1 -0
- package/dist/core/accesscontrol/DirectoryTimeTravelIndex.js +173 -0
- package/dist/core/accesscontrol/DirectoryTimeTravelIndex.js.map +1 -0
- package/dist/core/accesscontrol/directoryProjection.d.ts +26 -0
- package/dist/core/accesscontrol/directoryProjection.d.ts.map +1 -0
- package/dist/core/accesscontrol/directoryProjection.js +174 -0
- package/dist/core/accesscontrol/directoryProjection.js.map +1 -0
- package/dist/core/accesscontrol/evaluate.d.ts +77 -0
- package/dist/core/accesscontrol/evaluate.d.ts.map +1 -0
- package/dist/core/accesscontrol/evaluate.js +206 -0
- package/dist/core/accesscontrol/evaluate.js.map +1 -0
- package/dist/core/accesscontrol/grantKeys.d.ts +113 -0
- package/dist/core/accesscontrol/grantKeys.d.ts.map +1 -0
- package/dist/core/accesscontrol/grantKeys.js +302 -0
- package/dist/core/accesscontrol/grantKeys.js.map +1 -0
- package/dist/core/accesscontrol/keyBagReconciler.d.ts +22 -0
- package/dist/core/accesscontrol/keyBagReconciler.d.ts.map +1 -0
- package/dist/core/accesscontrol/keyBagReconciler.js +3 -0
- package/dist/core/accesscontrol/keyBagReconciler.js.map +1 -0
- package/dist/core/accesscontrol/materializationGuard.d.ts +89 -0
- package/dist/core/accesscontrol/materializationGuard.d.ts.map +1 -0
- package/dist/core/accesscontrol/materializationGuard.js +69 -0
- package/dist/core/accesscontrol/materializationGuard.js.map +1 -0
- package/dist/core/accesscontrol/quarantine.d.ts +65 -0
- package/dist/core/accesscontrol/quarantine.d.ts.map +1 -0
- package/dist/core/accesscontrol/quarantine.js +79 -0
- package/dist/core/accesscontrol/quarantine.js.map +1 -0
- package/dist/core/accesscontrol/receiptValidation.d.ts +113 -0
- package/dist/core/accesscontrol/receiptValidation.d.ts.map +1 -0
- package/dist/core/accesscontrol/receiptValidation.js +142 -0
- package/dist/core/accesscontrol/receiptValidation.js.map +1 -0
- package/dist/core/accesscontrol/timestamp/Ed25519WitnessProvider.d.ts +31 -0
- package/dist/core/accesscontrol/timestamp/Ed25519WitnessProvider.d.ts.map +1 -0
- package/dist/core/accesscontrol/timestamp/Ed25519WitnessProvider.js +66 -0
- package/dist/core/accesscontrol/timestamp/Ed25519WitnessProvider.js.map +1 -0
- package/dist/core/accesscontrol/timestamp/TimestampProvider.d.ts +79 -0
- package/dist/core/accesscontrol/timestamp/TimestampProvider.d.ts.map +1 -0
- package/dist/core/accesscontrol/timestamp/TimestampProvider.js +22 -0
- package/dist/core/accesscontrol/timestamp/TimestampProvider.js.map +1 -0
- package/dist/core/accesscontrol/types.d.ts +825 -0
- package/dist/core/accesscontrol/types.d.ts.map +1 -0
- package/dist/core/accesscontrol/types.js +534 -0
- package/dist/core/accesscontrol/types.js.map +1 -0
- package/dist/core/appendonlystores/InMemoryContentAddressedStore.d.ts +9 -0
- package/dist/core/appendonlystores/InMemoryContentAddressedStore.d.ts.map +1 -1
- package/dist/core/appendonlystores/InMemoryContentAddressedStore.js +54 -0
- package/dist/core/appendonlystores/InMemoryContentAddressedStore.js.map +1 -1
- package/dist/core/appendonlystores/network/AuthenticationService.d.ts +38 -2
- package/dist/core/appendonlystores/network/AuthenticationService.d.ts.map +1 -1
- package/dist/core/appendonlystores/network/AuthenticationService.js +199 -26
- package/dist/core/appendonlystores/network/AuthenticationService.js.map +1 -1
- package/dist/core/appendonlystores/network/NetworkTransport.d.ts +17 -3
- package/dist/core/appendonlystores/network/NetworkTransport.d.ts.map +1 -1
- package/dist/core/appendonlystores/network/types.d.ts +58 -4
- package/dist/core/appendonlystores/network/types.d.ts.map +1 -1
- package/dist/core/appendonlystores/network/types.js +4 -0
- package/dist/core/appendonlystores/network/types.js.map +1 -1
- package/dist/core/appendonlystores/scanUtils.d.ts +27 -0
- package/dist/core/appendonlystores/scanUtils.d.ts.map +1 -0
- package/dist/core/appendonlystores/scanUtils.js +87 -0
- package/dist/core/appendonlystores/scanUtils.js.map +1 -0
- package/dist/core/appendonlystores/types.d.ts +23 -2
- package/dist/core/appendonlystores/types.d.ts.map +1 -1
- package/dist/core/cache/CacheManager.d.ts +13 -3
- package/dist/core/cache/CacheManager.d.ts.map +1 -1
- package/dist/core/cache/CacheManager.js +7 -4
- package/dist/core/cache/CacheManager.js.map +1 -1
- package/dist/core/crypto/EntrySignature.d.ts +137 -0
- package/dist/core/crypto/EntrySignature.d.ts.map +1 -0
- package/dist/core/crypto/EntrySignature.js +210 -0
- package/dist/core/crypto/EntrySignature.js.map +1 -0
- package/dist/core/crypto/MindooDocSigner.d.ts.map +1 -1
- package/dist/core/crypto/MindooDocSigner.js +6 -2
- package/dist/core/crypto/MindooDocSigner.js.map +1 -1
- package/dist/core/crypto/WitnessReceipt.d.ts +137 -0
- package/dist/core/crypto/WitnessReceipt.d.ts.map +1 -0
- package/dist/core/crypto/WitnessReceipt.js +182 -0
- package/dist/core/crypto/WitnessReceipt.js.map +1 -0
- package/dist/core/crypto/encryptedFields.d.ts +27 -0
- package/dist/core/crypto/encryptedFields.d.ts.map +1 -0
- package/dist/core/crypto/encryptedFields.js +58 -0
- package/dist/core/crypto/encryptedFields.js.map +1 -0
- package/dist/core/crypto/pbkdf2Iterations.d.ts +12 -0
- package/dist/core/crypto/pbkdf2Iterations.d.ts.map +1 -1
- package/dist/core/crypto/pbkdf2Iterations.js +23 -0
- package/dist/core/crypto/pbkdf2Iterations.js.map +1 -1
- package/dist/core/crypto/privateKeyEncryption.d.ts.map +1 -1
- package/dist/core/crypto/privateKeyEncryption.js +3 -1
- package/dist/core/crypto/privateKeyEncryption.js.map +1 -1
- package/dist/core/databaseIdValidation.d.ts +1 -1
- package/dist/core/databaseIdValidation.d.ts.map +1 -1
- package/dist/core/databaseIdValidation.js +4 -4
- package/dist/core/databaseIdValidation.js.map +1 -1
- package/dist/core/errors.d.ts +26 -0
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +42 -1
- package/dist/core/errors.js.map +1 -1
- package/dist/core/expressions/builder.d.ts +102 -0
- package/dist/core/expressions/builder.d.ts.map +1 -0
- package/dist/core/expressions/builder.js +249 -0
- package/dist/core/expressions/builder.js.map +1 -0
- package/dist/core/expressions/evaluateExpression.d.ts +78 -0
- package/dist/core/expressions/evaluateExpression.d.ts.map +1 -0
- package/dist/core/expressions/evaluateExpression.js +386 -0
- package/dist/core/expressions/evaluateExpression.js.map +1 -0
- package/dist/core/expressions/formulaSource.d.ts +24 -0
- package/dist/core/expressions/formulaSource.d.ts.map +1 -0
- package/dist/core/expressions/formulaSource.js +765 -0
- package/dist/core/expressions/formulaSource.js.map +1 -0
- package/dist/core/expressions/index.d.ts +8 -0
- package/dist/core/expressions/index.d.ts.map +1 -0
- package/dist/core/expressions/index.js +46 -0
- package/dist/core/expressions/index.js.map +1 -0
- package/dist/core/expressions/metadata.d.ts +23 -0
- package/dist/core/expressions/metadata.d.ts.map +1 -0
- package/dist/core/expressions/metadata.js +191 -0
- package/dist/core/expressions/metadata.js.map +1 -0
- package/dist/core/expressions/types.d.ts +164 -0
- package/dist/core/expressions/types.d.ts.map +1 -0
- package/dist/core/expressions/types.js +3 -0
- package/dist/core/expressions/types.js.map +1 -0
- package/dist/core/extraction/types.d.ts +47 -0
- package/dist/core/extraction/types.d.ts.map +1 -0
- package/dist/core/extraction/types.js +42 -0
- package/dist/core/extraction/types.js.map +1 -0
- package/dist/core/index.d.ts +13 -3
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +113 -4
- package/dist/core/index.js.map +1 -1
- package/dist/core/indexing/createViewDataProvider.d.ts +84 -0
- package/dist/core/indexing/createViewDataProvider.d.ts.map +1 -0
- package/dist/core/indexing/createViewDataProvider.js +139 -0
- package/dist/core/indexing/createViewDataProvider.js.map +1 -0
- package/dist/core/indexing/fulltext/DocumentFullTextIndex.d.ts +180 -0
- package/dist/core/indexing/fulltext/DocumentFullTextIndex.d.ts.map +1 -0
- package/dist/core/indexing/fulltext/DocumentFullTextIndex.js +548 -0
- package/dist/core/indexing/fulltext/DocumentFullTextIndex.js.map +1 -0
- package/dist/core/indexing/fulltext/SearchEngineAdapter.d.ts +69 -0
- package/dist/core/indexing/fulltext/SearchEngineAdapter.d.ts.map +1 -0
- package/dist/core/indexing/fulltext/SearchEngineAdapter.js +183 -0
- package/dist/core/indexing/fulltext/SearchEngineAdapter.js.map +1 -0
- package/dist/core/indexing/fulltext/extractFulltextText.d.ts +34 -0
- package/dist/core/indexing/fulltext/extractFulltextText.d.ts.map +1 -0
- package/dist/core/indexing/fulltext/extractFulltextText.js +129 -0
- package/dist/core/indexing/fulltext/extractFulltextText.js.map +1 -0
- package/dist/core/indexing/fulltext/index.d.ts +5 -0
- package/dist/core/indexing/fulltext/index.d.ts.map +1 -0
- package/dist/core/indexing/fulltext/index.js +20 -0
- package/dist/core/indexing/fulltext/index.js.map +1 -0
- package/dist/core/indexing/fulltext/types.d.ts +168 -0
- package/dist/core/indexing/fulltext/types.d.ts.map +1 -0
- package/dist/core/indexing/fulltext/types.js +89 -0
- package/dist/core/indexing/fulltext/types.js.map +1 -0
- package/dist/core/indexing/summary/DocumentSummaryStore.d.ts +137 -0
- package/dist/core/indexing/summary/DocumentSummaryStore.d.ts.map +1 -0
- package/dist/core/indexing/summary/DocumentSummaryStore.js +438 -0
- package/dist/core/indexing/summary/DocumentSummaryStore.js.map +1 -0
- package/dist/core/indexing/summary/SummaryVirtualViewDataProvider.d.ts +72 -0
- package/dist/core/indexing/summary/SummaryVirtualViewDataProvider.d.ts.map +1 -0
- package/dist/core/indexing/summary/SummaryVirtualViewDataProvider.js +221 -0
- package/dist/core/indexing/summary/SummaryVirtualViewDataProvider.js.map +1 -0
- package/dist/core/indexing/summary/extractSummaryFields.d.ts +75 -0
- package/dist/core/indexing/summary/extractSummaryFields.d.ts.map +1 -0
- package/dist/core/indexing/summary/extractSummaryFields.js +266 -0
- package/dist/core/indexing/summary/extractSummaryFields.js.map +1 -0
- package/dist/core/indexing/summary/index.d.ts +5 -0
- package/dist/core/indexing/summary/index.d.ts.map +1 -0
- package/dist/core/indexing/summary/index.js +23 -0
- package/dist/core/indexing/summary/index.js.map +1 -0
- package/dist/core/indexing/summary/types.d.ts +119 -0
- package/dist/core/indexing/summary/types.d.ts.map +1 -0
- package/dist/core/indexing/summary/types.js +81 -0
- package/dist/core/indexing/summary/types.js.map +1 -0
- package/dist/core/indexing/virtualviews/IVirtualViewDataProvider.d.ts +47 -1
- package/dist/core/indexing/virtualviews/IVirtualViewDataProvider.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.d.ts +24 -8
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.js +96 -41
- package/dist/core/indexing/virtualviews/MindooDBVirtualViewDataProvider.js.map +1 -1
- package/dist/core/indexing/virtualviews/ViewEntrySortKey.d.ts +2 -0
- package/dist/core/indexing/virtualviews/ViewEntrySortKey.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/ViewEntrySortKey.js +8 -3
- package/dist/core/indexing/virtualviews/ViewEntrySortKey.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualView.d.ts +111 -5
- package/dist/core/indexing/virtualviews/VirtualView.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualView.js +266 -75
- package/dist/core/indexing/virtualviews/VirtualView.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewColumn.d.ts +12 -0
- package/dist/core/indexing/virtualviews/VirtualViewColumn.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewColumn.js +1 -0
- package/dist/core/indexing/virtualviews/VirtualViewColumn.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.d.ts +12 -0
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.js +66 -5
- package/dist/core/indexing/virtualviews/VirtualViewEntryData.js.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewFactory.d.ts +50 -6
- package/dist/core/indexing/virtualviews/VirtualViewFactory.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/VirtualViewFactory.js +54 -12
- package/dist/core/indexing/virtualviews/VirtualViewFactory.js.map +1 -1
- package/dist/core/indexing/virtualviews/index.d.ts +3 -3
- package/dist/core/indexing/virtualviews/index.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/index.js.map +1 -1
- package/dist/core/indexing/virtualviews/types.d.ts +6 -0
- package/dist/core/indexing/virtualviews/types.d.ts.map +1 -1
- package/dist/core/indexing/virtualviews/types.js +11 -3
- package/dist/core/indexing/virtualviews/types.js.map +1 -1
- package/dist/core/keys/KeyBag.d.ts +25 -0
- package/dist/core/keys/KeyBag.d.ts.map +1 -1
- package/dist/core/keys/KeyBag.js +37 -1
- package/dist/core/keys/KeyBag.js.map +1 -1
- package/dist/core/query/executeQuery.d.ts +18 -0
- package/dist/core/query/executeQuery.d.ts.map +1 -0
- package/dist/core/query/executeQuery.js +313 -0
- package/dist/core/query/executeQuery.js.map +1 -0
- package/dist/core/query/index.d.ts +5 -0
- package/dist/core/query/index.d.ts.map +1 -0
- package/dist/core/query/index.js +14 -0
- package/dist/core/query/index.js.map +1 -0
- package/dist/core/query/queryLive.d.ts +27 -0
- package/dist/core/query/queryLive.d.ts.map +1 -0
- package/dist/core/query/queryLive.js +104 -0
- package/dist/core/query/queryLive.js.map +1 -0
- package/dist/core/query/queryView.d.ts +156 -0
- package/dist/core/query/queryView.d.ts.map +1 -0
- package/dist/core/query/queryView.js +266 -0
- package/dist/core/query/queryView.js.map +1 -0
- package/dist/core/query/types.d.ts +121 -0
- package/dist/core/query/types.d.ts.map +1 -0
- package/dist/core/query/types.js +18 -0
- package/dist/core/query/types.js.map +1 -0
- package/dist/core/storeEntryTime.d.ts +57 -0
- package/dist/core/storeEntryTime.d.ts.map +1 -0
- package/dist/core/storeEntryTime.js +72 -0
- package/dist/core/storeEntryTime.js.map +1 -0
- package/dist/core/types.d.ts +1371 -34
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +13 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/uri/MindooURI.d.ts +53 -1
- package/dist/core/uri/MindooURI.d.ts.map +1 -1
- package/dist/core/uri/MindooURI.js +135 -0
- package/dist/core/uri/MindooURI.js.map +1 -1
- package/dist/core/utils/urlSafety.d.ts +46 -0
- package/dist/core/utils/urlSafety.d.ts.map +1 -0
- package/dist/core/utils/urlSafety.js +144 -0
- package/dist/core/utils/urlSafety.js.map +1 -0
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.d.ts +14 -0
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.d.ts.map +1 -1
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.js +95 -4
- package/dist/node/appendonlystores/BasicOnDiskContentAddressedStore.js.map +1 -1
- package/dist/node/server/CapabilityMatcher.d.ts +10 -0
- package/dist/node/server/CapabilityMatcher.d.ts.map +1 -1
- package/dist/node/server/CapabilityMatcher.js +14 -2
- package/dist/node/server/CapabilityMatcher.js.map +1 -1
- package/dist/node/server/MindooDBServer.d.ts.map +1 -1
- package/dist/node/server/MindooDBServer.js +117 -8
- package/dist/node/server/MindooDBServer.js.map +1 -1
- package/dist/node/server/PurgedDocRegistry.d.ts +32 -0
- package/dist/node/server/PurgedDocRegistry.d.ts.map +1 -0
- package/dist/node/server/PurgedDocRegistry.js +94 -0
- package/dist/node/server/PurgedDocRegistry.js.map +1 -0
- package/dist/node/server/ServerSync.d.ts.map +1 -1
- package/dist/node/server/ServerSync.js +5 -2
- package/dist/node/server/ServerSync.js.map +1 -1
- package/dist/node/server/SystemAdminAuth.d.ts +10 -0
- package/dist/node/server/SystemAdminAuth.d.ts.map +1 -1
- package/dist/node/server/SystemAdminAuth.js +25 -0
- package/dist/node/server/SystemAdminAuth.js.map +1 -1
- package/dist/node/server/TenantManager.d.ts +95 -1
- package/dist/node/server/TenantManager.d.ts.map +1 -1
- package/dist/node/server/TenantManager.js +382 -15
- package/dist/node/server/TenantManager.js.map +1 -1
- package/dist/node/server/config.d.ts.map +1 -1
- package/dist/node/server/config.js +5 -1
- package/dist/node/server/config.js.map +1 -1
- package/dist/node/server/types.d.ts +29 -3
- package/dist/node/server/types.d.ts.map +1 -1
- package/dist/node/server/types.js +12 -0
- package/dist/node/server/types.js.map +1 -1
- package/dist/node/server/validation.d.ts +6 -0
- package/dist/node/server/validation.d.ts.map +1 -1
- package/dist/node/server/validation.js +7 -1
- package/dist/node/server/validation.js.map +1 -1
- package/package.json +4 -3
- package/dist/appendonlystores/BasicOnDiskContentAddressedStoreFactory.d.ts +0 -8
- package/dist/appendonlystores/BasicOnDiskContentAddressedStoreFactory.d.ts.map +0 -1
- package/dist/appendonlystores/BasicOnDiskContentAddressedStoreFactory.js +0 -16
- package/dist/appendonlystores/BasicOnDiskContentAddressedStoreFactory.js.map +0 -1
- package/dist/core/appendonlystores/BasicOnDiskContentAddressedStore.d.ts +0 -68
- package/dist/core/appendonlystores/BasicOnDiskContentAddressedStore.d.ts.map +0 -1
- package/dist/core/appendonlystores/BasicOnDiskContentAddressedStore.js +0 -659
- package/dist/core/appendonlystores/BasicOnDiskContentAddressedStore.js.map +0 -1
- package/dist/node/attachments/writeAttachmentToFile.d.ts +0 -15
- package/dist/node/attachments/writeAttachmentToFile.d.ts.map +0 -1
- package/dist/node/attachments/writeAttachmentToFile.js +0 -73
- package/dist/node/attachments/writeAttachmentToFile.js.map +0 -1
|
@@ -5,6 +5,7 @@ exports.BaseMindooDB = exports.CUSTOM_DOC_ID_INITIAL_CHANGE_BYTES = void 0;
|
|
|
5
5
|
// React Native: native Rust (react-native-automerge-generated)
|
|
6
6
|
// Browser/Node.js: WASM (@automerge/automerge/slim)
|
|
7
7
|
const automerge_adapter_1 = require("./automerge-adapter");
|
|
8
|
+
const storeEntryTime_1 = require("./storeEntryTime");
|
|
8
9
|
const uuid_1 = require("uuid");
|
|
9
10
|
const types_1 = require("./types");
|
|
10
11
|
const types_2 = require("./appendonlystores/types");
|
|
@@ -15,8 +16,20 @@ const DocumentConflictAnalysis_1 = require("./DocumentConflictAnalysis");
|
|
|
15
16
|
const AttachmentReadPlanner_1 = require("./appendonlystores/AttachmentReadPlanner");
|
|
16
17
|
const idGeneration_1 = require("./utils/idGeneration");
|
|
17
18
|
const errors_1 = require("./errors");
|
|
19
|
+
const EntrySignature_1 = require("./crypto/EntrySignature");
|
|
20
|
+
const materializationGuard_1 = require("./accesscontrol/materializationGuard");
|
|
21
|
+
const Ed25519WitnessProvider_1 = require("./accesscontrol/timestamp/Ed25519WitnessProvider");
|
|
22
|
+
const AccessDeniedError_1 = require("./accesscontrol/AccessDeniedError");
|
|
18
23
|
const logging_1 = require("./logging");
|
|
19
24
|
const databaseIdValidation_1 = require("./databaseIdValidation");
|
|
25
|
+
const DocumentSummaryStore_1 = require("./indexing/summary/DocumentSummaryStore");
|
|
26
|
+
const types_3 = require("./indexing/summary/types");
|
|
27
|
+
const DocumentFullTextIndex_1 = require("./indexing/fulltext/DocumentFullTextIndex");
|
|
28
|
+
const types_4 = require("./indexing/fulltext/types");
|
|
29
|
+
const types_5 = require("./extraction/types");
|
|
30
|
+
const executeQuery_1 = require("./query/executeQuery");
|
|
31
|
+
const queryView_1 = require("./query/queryView");
|
|
32
|
+
const queryLive_1 = require("./query/queryLive");
|
|
20
33
|
/**
|
|
21
34
|
* Default chunk size for attachments: 256KB
|
|
22
35
|
*/
|
|
@@ -129,18 +142,73 @@ function getCustomIdInitialChangeBytes() {
|
|
|
129
142
|
*/
|
|
130
143
|
class BaseMindooDB {
|
|
131
144
|
constructor(tenant, store, attachmentStore, attachmentConfig, documentCacheConfig, snapshotConfig, adminOnlyDb = false, logger, performanceCallback, timeTravelDate = null) {
|
|
145
|
+
/**
|
|
146
|
+
* Per-tenant quarantine/audit log of entries rejected during materialization
|
|
147
|
+
* (docs/accesscontrol.md §10). In-memory; surfaced via {@link getQuarantineLog}
|
|
148
|
+
* for Haven's audit view.
|
|
149
|
+
*/
|
|
150
|
+
this.quarantineLog = [];
|
|
151
|
+
/**
|
|
152
|
+
* Short-TTL cache of "is access control active for this tenant?" so the
|
|
153
|
+
* materialization fast-path does not re-query the directory on every load.
|
|
154
|
+
*/
|
|
155
|
+
this.aclActiveCache = null;
|
|
156
|
+
/**
|
|
157
|
+
* Per-document cache of the original creator's signing public key (the
|
|
158
|
+
* `doc_create` entry's `createdByPublicKey`). Used by the client write
|
|
159
|
+
* prechecks to resolve `$author` (`isAuthor`) without re-scanning metadata on
|
|
160
|
+
* every change/delete. Populated on create and lazily on first lookup.
|
|
161
|
+
*/
|
|
162
|
+
this.creatorKeyCache = new Map();
|
|
132
163
|
// Internal changefeed index: sorted by (changeSeq, docId) for deterministic iteration.
|
|
133
164
|
// lastModified remains available for UX metadata but is not the primary cursor key.
|
|
134
165
|
this.index = [];
|
|
135
166
|
// Lookup map for O(1) access to index entries by docId
|
|
136
167
|
this.indexLookup = new Map(); // Map<docId, arrayIndex>
|
|
168
|
+
/**
|
|
169
|
+
* Lazily-deferred repair marker for {@link indexLookup}: when set, positions
|
|
170
|
+
* stored in the map for array indices >= this value may be stale (entries
|
|
171
|
+
* shifted by an {@link updateIndex} splice). Key presence is always exact.
|
|
172
|
+
* Reads go through {@link getDocIndexPosition}, which validates a hit
|
|
173
|
+
* against {@link index} and triggers {@link flushIndexLookupPatch} only when
|
|
174
|
+
* the position actually moved. This turns the per-update O(index size)
|
|
175
|
+
* lookup rewrite into a single deferred pass — bulk syncs that append at the
|
|
176
|
+
* index tail never pay it at all.
|
|
177
|
+
*/
|
|
178
|
+
this.indexLookupStaleFrom = null;
|
|
137
179
|
this.nextChangeSeq = 1;
|
|
138
180
|
// Cache of loaded documents: Map<docId, InternalDoc>
|
|
139
181
|
this.docCache = new Map();
|
|
140
182
|
this.wrappedInternalDocs = new WeakMap();
|
|
183
|
+
/**
|
|
184
|
+
* Per-document count of local writes since the last snapshot eligibility
|
|
185
|
+
* check that scanned the store. Lets {@link maybeWriteSnapshotForDocument}
|
|
186
|
+
* skip the per-doc metadata scan entirely while a document is still far
|
|
187
|
+
* below {@link snapshotMinChanges}. Absent key = unknown (scan once).
|
|
188
|
+
*/
|
|
189
|
+
this.writesSinceSnapshotCheck = new Map();
|
|
141
190
|
// Track which entry IDs we've already processed
|
|
142
191
|
this.processedEntryIds = [];
|
|
143
192
|
this.processedEntryCursor = null;
|
|
193
|
+
/**
|
|
194
|
+
* In-memory LRU of per-document merged state used by the revision-grain
|
|
195
|
+
* changefeed ({@link iterateRevisionsInternal}). Each entry holds a document's
|
|
196
|
+
* fully-merged Automerge doc together with the exact set of replay entry ids
|
|
197
|
+
* that merge covers, so an incremental feed advance can seed the next fold
|
|
198
|
+
* from the cached state (applying only the newly-arrived suffix) instead of
|
|
199
|
+
* re-decrypting and re-folding the document's whole history. Bounded by
|
|
200
|
+
* {@link revisionFeedCacheLimit}; least-recently-used docs are evicted and
|
|
201
|
+
* reconstructed from the best snapshot on the next touch.
|
|
202
|
+
*/
|
|
203
|
+
this.revisionFeedDocCache = new Map();
|
|
204
|
+
this.revisionFeedCacheLimit = 256;
|
|
205
|
+
/**
|
|
206
|
+
* Window size for batching encrypted-payload loads while folding a document's
|
|
207
|
+
* revision suffix ({@link foldDocRevisions}): one {@link ContentAddressedStore.getEntries}
|
|
208
|
+
* round trip per window instead of one per revision, while bounding peak
|
|
209
|
+
* memory and response size on a snapshot-less cold-start re-fold.
|
|
210
|
+
*/
|
|
211
|
+
this.revisionFoldLoadWindow = 256;
|
|
144
212
|
// Index: automergeHash -> entryId for each document
|
|
145
213
|
// Used for resolving Automerge dependency hashes to entry IDs
|
|
146
214
|
this.automergeHashToEntryId = new Map(); // Map<docId, Map<automergeHash, entryId>>
|
|
@@ -150,8 +218,65 @@ class BaseMindooDB {
|
|
|
150
218
|
// Local cache support
|
|
151
219
|
this.cacheManager = null;
|
|
152
220
|
this.cachePrefix = null;
|
|
221
|
+
/**
|
|
222
|
+
* Lazily created document summary buffer backing ad-hoc queries and
|
|
223
|
+
* summary-based views. `null` until {@link getSummaryStore} is first
|
|
224
|
+
* called, so databases that never query pay no cost.
|
|
225
|
+
*/
|
|
226
|
+
this.summaryStore = null;
|
|
227
|
+
/**
|
|
228
|
+
* Lazily created full-text index (opt-in via `fulltextSetup` or an
|
|
229
|
+
* explicit config). `null` until {@link getFullTextIndex} is first
|
|
230
|
+
* called or the setup document enables indexing.
|
|
231
|
+
*/
|
|
232
|
+
this.fulltextIndex = null;
|
|
233
|
+
/**
|
|
234
|
+
* Attachment text extractors registered by the host environment (see
|
|
235
|
+
* {@link registerAttachmentTextExtractor}). Consulted lazily by the
|
|
236
|
+
* full-text index while documents with attachments are indexed.
|
|
237
|
+
*/
|
|
238
|
+
this.attachmentTextExtractors = [];
|
|
239
|
+
// ---------------------------------------------------------------------------
|
|
240
|
+
// Change notification state (see addChangeListener). `updateIndex` is the
|
|
241
|
+
// common choke point of all mutation paths, so it only records pending
|
|
242
|
+
// docIds; emission is coalesced per sync batch / event-loop turn.
|
|
243
|
+
// ---------------------------------------------------------------------------
|
|
244
|
+
this.changeListeners = new Set();
|
|
245
|
+
this.pendingChangeNotifications = new Map();
|
|
246
|
+
this.changeNotifyTimer = null;
|
|
247
|
+
/** While > 0, timer-based emission is suppressed (e.g. during a sync batch). */
|
|
248
|
+
this.changeNotificationHolds = 0;
|
|
249
|
+
// ---------------------------------------------------------------------------
|
|
250
|
+
// Summary auto-follow (see setSummaryAutoUpdateEnabled): keeps the summary
|
|
251
|
+
// buffer current as changes arrive (local writes and sync batches), so the
|
|
252
|
+
// first query after receiving thousands of documents doesn't stall on a
|
|
253
|
+
// full catch-up run.
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
this.summaryAutoUpdateEnabled = true;
|
|
256
|
+
this.summaryAutoUpdateRunning = false;
|
|
257
|
+
this.summaryAutoUpdatePending = false;
|
|
258
|
+
/** Whether the setup document was already probed for auto-activation. */
|
|
259
|
+
this.summarySetupProbed = false;
|
|
260
|
+
// ---------------------------------------------------------------------------
|
|
261
|
+
// Full-text auto-follow: mirrors the summary auto-follow, but the index
|
|
262
|
+
// is opt-in — auto-activation only happens when the setup document has
|
|
263
|
+
// `fulltextSetup.enabled: true`.
|
|
264
|
+
// ---------------------------------------------------------------------------
|
|
265
|
+
this.fulltextAutoUpdateEnabled = true;
|
|
266
|
+
this.fulltextAutoUpdateRunning = false;
|
|
267
|
+
this.fulltextAutoUpdatePending = false;
|
|
268
|
+
/** Whether the setup document was already probed for FTS auto-activation. */
|
|
269
|
+
this.fulltextSetupProbed = false;
|
|
153
270
|
this.dirtyDocIds = new Set();
|
|
154
271
|
this.cacheMetaDirty = false;
|
|
272
|
+
/**
|
|
273
|
+
* Per-document fingerprint (`changeSeq:automergeHeads`) of the state last
|
|
274
|
+
* written to the L2 cache by {@link flushDirtyDocToCache}. Lets the periodic
|
|
275
|
+
* flush skip the expensive `Automerge.save` + L2 write for documents that
|
|
276
|
+
* were marked dirty but whose persisted state is already current (e.g. docs
|
|
277
|
+
* that were merely loaded and cached, not changed).
|
|
278
|
+
*/
|
|
279
|
+
this.lastFlushedDocState = new Map();
|
|
155
280
|
/**
|
|
156
281
|
* Tenant KeyBag fingerprint observed at the time the in-memory index
|
|
157
282
|
* was last reconciled with the underlying store.
|
|
@@ -221,6 +346,25 @@ class BaseMindooDB {
|
|
|
221
346
|
logger ||
|
|
222
347
|
new logging_1.MindooLogger((0, logging_1.getDefaultLogLevel)(), "BaseMindooDB", true);
|
|
223
348
|
this.performanceCallback = performanceCallback;
|
|
349
|
+
// Summary auto-follow: piggybacks on the coalesced change events (one
|
|
350
|
+
// per sync batch / event-loop turn). Costs a docId check per event for
|
|
351
|
+
// databases that never use summaries.
|
|
352
|
+
this.addChangeListener((event) => {
|
|
353
|
+
this.handleChangesForSummaryAutoUpdate(event.changes);
|
|
354
|
+
});
|
|
355
|
+
// Full-text auto-follow: same coalesced events, opt-in activation.
|
|
356
|
+
this.addChangeListener((event) => {
|
|
357
|
+
this.handleChangesForFulltextAutoUpdate(event.changes);
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Whether debug logging is enabled. Used to guard hot-path debug statements
|
|
362
|
+
* whose message interpolation is itself expensive (e.g.
|
|
363
|
+
* `JSON.stringify(Automerge.getHeads(...))` — a WASM call plus stringify per
|
|
364
|
+
* document), so bulk operations don't pay for logging that is discarded.
|
|
365
|
+
*/
|
|
366
|
+
isDebugEnabled() {
|
|
367
|
+
return this.logger.isLevelEnabled(logging_1.LogLevel.DEBUG);
|
|
224
368
|
}
|
|
225
369
|
/**
|
|
226
370
|
* Get the admin public key from the tenant.
|
|
@@ -276,6 +420,503 @@ class BaseMindooDB {
|
|
|
276
420
|
const cacheIdentity = this.store.getCacheIdentity?.() ?? this.store.getId();
|
|
277
421
|
this.cachePrefix = `${this.tenant.getId()}/${cacheIdentity}`;
|
|
278
422
|
cacheManager.register(this);
|
|
423
|
+
// A summary store created before the cache manager was attached picks
|
|
424
|
+
// up persistence now.
|
|
425
|
+
this.summaryStore?.attachCache(cacheManager, `${this.getCachePrefix()}/summary`);
|
|
426
|
+
this.fulltextIndex?.attachCache(cacheManager, `${this.getCachePrefix()}/fulltext`);
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Get (lazily creating) the document summary buffer for this database.
|
|
430
|
+
*
|
|
431
|
+
* The summary is a changefeed-maintained map of queryable field values
|
|
432
|
+
* per document — the substrate for `query()`/`queryView()` (see
|
|
433
|
+
* docs/adhoc-queries.md). Passing a `config` on a later call replaces the
|
|
434
|
+
* configuration; if it differs from the active one, a resumable backfill
|
|
435
|
+
* re-extracts all documents on the next `update()` (documents are never
|
|
436
|
+
* rewritten).
|
|
437
|
+
*/
|
|
438
|
+
getSummaryStore(config) {
|
|
439
|
+
if (!this.summaryStore) {
|
|
440
|
+
this.summaryStore = new DocumentSummaryStore_1.DocumentSummaryStore(this, config);
|
|
441
|
+
if (this.cacheManager) {
|
|
442
|
+
this.summaryStore.attachCache(this.cacheManager, `${this.getCachePrefix()}/summary`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
else if (config !== undefined) {
|
|
446
|
+
this.summaryStore.setConfig(config);
|
|
447
|
+
}
|
|
448
|
+
return this.summaryStore;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Read the summary configuration stored in the synced setup document
|
|
452
|
+
* (`dbsetup`, field `summarySetup`) — the "design document" that lets
|
|
453
|
+
* users/apps (e.g. the Haven UI) configure the summary fields in one
|
|
454
|
+
* place for all replicas. Returns `null` when no setup document or no
|
|
455
|
+
* configuration exists.
|
|
456
|
+
*/
|
|
457
|
+
async getSummarySetup() {
|
|
458
|
+
try {
|
|
459
|
+
const doc = await this.getDocument(types_3.DB_SETUP_DOC_ID);
|
|
460
|
+
return (0, types_3.sanitizeSummaryConfig)(doc.getData()[types_3.SUMMARY_SETUP_FIELD]) ?? null;
|
|
461
|
+
}
|
|
462
|
+
catch (error) {
|
|
463
|
+
if (error instanceof errors_1.DocumentNotFoundError || error instanceof errors_1.DocumentDeletedError) {
|
|
464
|
+
return null;
|
|
465
|
+
}
|
|
466
|
+
throw error;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Write the summary configuration into the synced setup document
|
|
471
|
+
* (creating `dbsetup` idempotently when missing; custom-ID documents
|
|
472
|
+
* share seeded Automerge ancestry, so concurrent creates on different
|
|
473
|
+
* replicas merge). Passing `null` removes the configuration (replicas
|
|
474
|
+
* fall back to the default auto-include config).
|
|
475
|
+
*
|
|
476
|
+
* Summary stores without an explicit code-provided configuration pick
|
|
477
|
+
* the change up through the changefeed and re-extract via a resumable
|
|
478
|
+
* backfill — locally and, after sync, on every other replica.
|
|
479
|
+
*/
|
|
480
|
+
async setSummarySetup(config) {
|
|
481
|
+
const doc = await this.createDocument({ id: types_3.DB_SETUP_DOC_ID });
|
|
482
|
+
await this.changeDoc(doc, (d) => {
|
|
483
|
+
const data = d.getData();
|
|
484
|
+
if (config === null) {
|
|
485
|
+
delete data[types_3.SUMMARY_SETUP_FIELD];
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
data[types_3.SUMMARY_SETUP_FIELD] = (0, types_3.sanitizeSummaryConfig)(config) ?? {};
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Enable/disable summary auto-follow (default: enabled). When enabled,
|
|
494
|
+
* the summary buffer is brought up to date in the background after every
|
|
495
|
+
* coalesced change event — one catch-up run per sync batch, one per
|
|
496
|
+
* event-loop turn for local writes — so queries don't stall on a full
|
|
497
|
+
* catch-up after receiving many documents. Compared to the network I/O
|
|
498
|
+
* of a sync the incremental extraction is cheap; disable it for
|
|
499
|
+
* bulk-import scenarios where even that is unwanted (the next query
|
|
500
|
+
* catches up lazily as before).
|
|
501
|
+
*/
|
|
502
|
+
setSummaryAutoUpdateEnabled(enabled) {
|
|
503
|
+
this.summaryAutoUpdateEnabled = enabled;
|
|
504
|
+
if (enabled && this.summaryStore) {
|
|
505
|
+
this.scheduleSummaryAutoUpdate();
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
/**
|
|
509
|
+
* Change-event hook for summary auto-follow. While no summary store
|
|
510
|
+
* exists, the setup document decides whether to activate one: it is
|
|
511
|
+
* probed on the first event (covers reopening a database whose `dbsetup`
|
|
512
|
+
* synced earlier) and re-checked whenever a `dbsetup` change arrives
|
|
513
|
+
* (covers the configuration being created/updated locally or via sync).
|
|
514
|
+
*/
|
|
515
|
+
handleChangesForSummaryAutoUpdate(changes) {
|
|
516
|
+
if (!this.summaryAutoUpdateEnabled) {
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
if (this.summaryStore) {
|
|
520
|
+
this.scheduleSummaryAutoUpdate();
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
// Never auto-activate on system databases: the directory has its own
|
|
524
|
+
// indexing (DirectoryTimeTravelIndex) and its change events interleave
|
|
525
|
+
// with access-control evaluation; admin-only databases hold no user
|
|
526
|
+
// data worth summarizing. Explicit getSummaryStore() calls still work.
|
|
527
|
+
if (this._isAdminOnlyDb || this.store.getId() === "directory") {
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
const hasSetupChange = changes.some((change) => change.docId === types_3.DB_SETUP_DOC_ID);
|
|
531
|
+
if (!hasSetupChange && this.summarySetupProbed) {
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
this.summarySetupProbed = true;
|
|
535
|
+
void this.activateSummaryFromSetupDoc();
|
|
536
|
+
}
|
|
537
|
+
/** Create the summary store when the setup document configures one. */
|
|
538
|
+
async activateSummaryFromSetupDoc() {
|
|
539
|
+
try {
|
|
540
|
+
const config = await this.getSummarySetup();
|
|
541
|
+
if (config === null || this.summaryStore || !this.summaryAutoUpdateEnabled) {
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
// No explicit config: the store adopts the setup-document config on
|
|
545
|
+
// its first update run (seedConfigFromSetupDoc).
|
|
546
|
+
this.getSummaryStore();
|
|
547
|
+
this.scheduleSummaryAutoUpdate();
|
|
548
|
+
}
|
|
549
|
+
catch (error) {
|
|
550
|
+
this.logger.warn(`Summary auto-activation failed: ${error}`);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
/**
|
|
554
|
+
* Run a background summary catch-up, coalescing further requests into
|
|
555
|
+
* one follow-up run (mirrors the live-view scheduling in VirtualView).
|
|
556
|
+
* Errors are logged and never propagate into write/sync paths.
|
|
557
|
+
*/
|
|
558
|
+
scheduleSummaryAutoUpdate() {
|
|
559
|
+
if (this.summaryAutoUpdateRunning) {
|
|
560
|
+
this.summaryAutoUpdatePending = true;
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
this.summaryAutoUpdateRunning = true;
|
|
564
|
+
void (async () => {
|
|
565
|
+
try {
|
|
566
|
+
do {
|
|
567
|
+
this.summaryAutoUpdatePending = false;
|
|
568
|
+
await this.summaryStore?.update();
|
|
569
|
+
} while (this.summaryAutoUpdatePending && this.summaryAutoUpdateEnabled);
|
|
570
|
+
}
|
|
571
|
+
catch (error) {
|
|
572
|
+
this.logger.warn(`Summary auto-update failed: ${error}`);
|
|
573
|
+
}
|
|
574
|
+
finally {
|
|
575
|
+
this.summaryAutoUpdateRunning = false;
|
|
576
|
+
}
|
|
577
|
+
})();
|
|
578
|
+
}
|
|
579
|
+
// ---------------------------------------------------------------------------
|
|
580
|
+
// Full-text index (see docs/fulltext-search.md)
|
|
581
|
+
// ---------------------------------------------------------------------------
|
|
582
|
+
/**
|
|
583
|
+
* Get (lazily creating) the full-text index for this database.
|
|
584
|
+
*
|
|
585
|
+
* The index is a changefeed-maintained, encrypted-at-rest search index
|
|
586
|
+
* over extracted document text — configured via the synced setup
|
|
587
|
+
* document (`dbsetup`, field `fulltextSetup`) or an explicit `config`.
|
|
588
|
+
* Passing a `config` on a later call replaces the configuration; if it
|
|
589
|
+
* differs from the active one, the engine is recreated and a resumable
|
|
590
|
+
* backfill re-indexes all documents on the next `update()`.
|
|
591
|
+
*
|
|
592
|
+
* Note that indexing is OPT-IN: without `enabled: true` (in the setup
|
|
593
|
+
* document or the explicit config) the index stays empty and
|
|
594
|
+
* {@link searchText} throws.
|
|
595
|
+
*/
|
|
596
|
+
getFullTextIndex(config) {
|
|
597
|
+
if (!this.fulltextIndex) {
|
|
598
|
+
this.fulltextIndex = new DocumentFullTextIndex_1.DocumentFullTextIndex(this, config, {
|
|
599
|
+
getAttachmentExtractors: () => this.attachmentTextExtractors,
|
|
600
|
+
});
|
|
601
|
+
if (this.cacheManager) {
|
|
602
|
+
this.fulltextIndex.attachCache(this.cacheManager, `${this.getCachePrefix()}/fulltext`);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
else if (config !== undefined) {
|
|
606
|
+
this.fulltextIndex.setConfig(config);
|
|
607
|
+
}
|
|
608
|
+
return this.fulltextIndex;
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Read the full-text configuration stored in the synced setup document
|
|
612
|
+
* (`dbsetup`, field `fulltextSetup`). Returns `null` when no setup
|
|
613
|
+
* document or no configuration exists.
|
|
614
|
+
*/
|
|
615
|
+
async getFulltextSetup() {
|
|
616
|
+
try {
|
|
617
|
+
const doc = await this.getDocument(types_3.DB_SETUP_DOC_ID);
|
|
618
|
+
return (0, types_4.sanitizeFulltextConfig)(doc.getData()[types_4.FULLTEXT_SETUP_FIELD]) ?? null;
|
|
619
|
+
}
|
|
620
|
+
catch (error) {
|
|
621
|
+
if (error instanceof errors_1.DocumentNotFoundError || error instanceof errors_1.DocumentDeletedError) {
|
|
622
|
+
return null;
|
|
623
|
+
}
|
|
624
|
+
throw error;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Write the full-text configuration into the synced setup document
|
|
629
|
+
* (creating `dbsetup` idempotently when missing). Passing `null`
|
|
630
|
+
* removes the configuration (indexing falls back to disabled).
|
|
631
|
+
*
|
|
632
|
+
* Idempotent: when the sanitized config equals what is already stored,
|
|
633
|
+
* no change is written — apps can safely call this on every start
|
|
634
|
+
* without producing changefeed/sync churn.
|
|
635
|
+
*
|
|
636
|
+
* Full-text indexes without an explicit code-provided configuration
|
|
637
|
+
* pick the change up through the changefeed and re-index via a
|
|
638
|
+
* resumable backfill — locally and, after sync, on every other replica.
|
|
639
|
+
*/
|
|
640
|
+
async setFulltextSetup(config) {
|
|
641
|
+
const sanitized = config === null ? null : (0, types_4.sanitizeFulltextConfig)(config) ?? {};
|
|
642
|
+
const current = await this.getFulltextSetup();
|
|
643
|
+
if (JSON.stringify(current) === JSON.stringify(sanitized)) {
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
const doc = await this.createDocument({ id: types_3.DB_SETUP_DOC_ID });
|
|
647
|
+
await this.changeDoc(doc, (d) => {
|
|
648
|
+
const data = d.getData();
|
|
649
|
+
if (sanitized === null) {
|
|
650
|
+
delete data[types_4.FULLTEXT_SETUP_FIELD];
|
|
651
|
+
}
|
|
652
|
+
else {
|
|
653
|
+
data[types_4.FULLTEXT_SETUP_FIELD] = sanitized;
|
|
654
|
+
}
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
658
|
+
* Read the attachment extraction configuration stored in the synced
|
|
659
|
+
* setup document (`dbsetup`, field `extractionSetup`). Configures
|
|
660
|
+
* host-side extraction services (e.g. Haven's OCR service) that persist
|
|
661
|
+
* results at the attachment entry via
|
|
662
|
+
* {@link MindooDoc.setAttachmentExtractedText}. Returns `null` when no
|
|
663
|
+
* setup document or no configuration exists.
|
|
664
|
+
*/
|
|
665
|
+
async getExtractionSetup() {
|
|
666
|
+
try {
|
|
667
|
+
const doc = await this.getDocument(types_3.DB_SETUP_DOC_ID);
|
|
668
|
+
return (0, types_5.sanitizeExtractionConfig)(doc.getData()[types_5.EXTRACTION_SETUP_FIELD]) ?? null;
|
|
669
|
+
}
|
|
670
|
+
catch (error) {
|
|
671
|
+
if (error instanceof errors_1.DocumentNotFoundError || error instanceof errors_1.DocumentDeletedError) {
|
|
672
|
+
return null;
|
|
673
|
+
}
|
|
674
|
+
throw error;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
/**
|
|
678
|
+
* Write the attachment extraction configuration into the synced setup
|
|
679
|
+
* document (creating `dbsetup` idempotently when missing). Passing
|
|
680
|
+
* `null` removes the configuration (extraction services stay idle).
|
|
681
|
+
*
|
|
682
|
+
* Idempotent: when the sanitized config equals what is already stored,
|
|
683
|
+
* no change is written — services can safely call this on every start
|
|
684
|
+
* without producing changefeed/sync churn.
|
|
685
|
+
*/
|
|
686
|
+
async setExtractionSetup(config) {
|
|
687
|
+
const sanitized = config === null ? null : (0, types_5.sanitizeExtractionConfig)(config) ?? {};
|
|
688
|
+
const current = await this.getExtractionSetup();
|
|
689
|
+
if (JSON.stringify(current) === JSON.stringify(sanitized)) {
|
|
690
|
+
return;
|
|
691
|
+
}
|
|
692
|
+
const doc = await this.createDocument({ id: types_3.DB_SETUP_DOC_ID });
|
|
693
|
+
await this.changeDoc(doc, (d) => {
|
|
694
|
+
const data = d.getData();
|
|
695
|
+
if (sanitized === null) {
|
|
696
|
+
delete data[types_5.EXTRACTION_SETUP_FIELD];
|
|
697
|
+
}
|
|
698
|
+
else {
|
|
699
|
+
data[types_5.EXTRACTION_SETUP_FIELD] = sanitized;
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
/**
|
|
704
|
+
* Register an attachment text extractor for full-text indexing (see
|
|
705
|
+
* {@link AttachmentTextExtractor}). Host environments call this when
|
|
706
|
+
* opening the database (e.g. Haven registers text/PDF/Office extractors
|
|
707
|
+
* from its preview parsers). Extraction only happens for databases
|
|
708
|
+
* whose `fulltextSetup` has `attachments: true`.
|
|
709
|
+
*
|
|
710
|
+
* @returns An unregister function.
|
|
711
|
+
*/
|
|
712
|
+
registerAttachmentTextExtractor(extractor) {
|
|
713
|
+
this.attachmentTextExtractors.push(extractor);
|
|
714
|
+
return () => {
|
|
715
|
+
const index = this.attachmentTextExtractors.indexOf(extractor);
|
|
716
|
+
if (index >= 0) {
|
|
717
|
+
this.attachmentTextExtractors.splice(index, 1);
|
|
718
|
+
}
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* Full-text search over the document full-text index (see
|
|
723
|
+
* docs/fulltext-search.md). Brings the index up to date first, then
|
|
724
|
+
* returns matching documents best-score-first. Results are
|
|
725
|
+
* device-specific under E2EE: documents whose decryption keys are
|
|
726
|
+
* missing from the local KeyBag are not indexed and cannot match.
|
|
727
|
+
*
|
|
728
|
+
* @throws Error when full-text indexing is not enabled for this
|
|
729
|
+
* database (see {@link setFulltextSetup} / {@link getFullTextIndex}).
|
|
730
|
+
*/
|
|
731
|
+
async searchText(query, options) {
|
|
732
|
+
return this.getFullTextIndex().search(query, options);
|
|
733
|
+
}
|
|
734
|
+
/**
|
|
735
|
+
* Enable/disable full-text auto-follow (default: enabled). Mirrors
|
|
736
|
+
* {@link setSummaryAutoUpdateEnabled}: when enabled, an ACTIVE full-text
|
|
737
|
+
* index catches up in the background after every coalesced change
|
|
738
|
+
* event, so the first search after a large sync doesn't stall.
|
|
739
|
+
* Databases without an enabled `fulltextSetup` never pay any cost.
|
|
740
|
+
*/
|
|
741
|
+
setFulltextAutoUpdateEnabled(enabled) {
|
|
742
|
+
this.fulltextAutoUpdateEnabled = enabled;
|
|
743
|
+
if (enabled && this.fulltextIndex) {
|
|
744
|
+
this.scheduleFulltextAutoUpdate();
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
/**
|
|
748
|
+
* Change-event hook for full-text auto-follow. While no index exists,
|
|
749
|
+
* the setup document decides whether to activate one — probed on the
|
|
750
|
+
* first event and re-checked whenever a `dbsetup` change arrives
|
|
751
|
+
* (mirrors {@link handleChangesForSummaryAutoUpdate}).
|
|
752
|
+
*/
|
|
753
|
+
handleChangesForFulltextAutoUpdate(changes) {
|
|
754
|
+
if (!this.fulltextAutoUpdateEnabled) {
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
if (this.fulltextIndex) {
|
|
758
|
+
this.scheduleFulltextAutoUpdate();
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
// Same system-database exclusion as the summary auto-activation.
|
|
762
|
+
if (this._isAdminOnlyDb || this.store.getId() === "directory") {
|
|
763
|
+
return;
|
|
764
|
+
}
|
|
765
|
+
const hasSetupChange = changes.some((change) => change.docId === types_3.DB_SETUP_DOC_ID);
|
|
766
|
+
if (!hasSetupChange && this.fulltextSetupProbed) {
|
|
767
|
+
return;
|
|
768
|
+
}
|
|
769
|
+
this.fulltextSetupProbed = true;
|
|
770
|
+
void this.activateFulltextFromSetupDoc();
|
|
771
|
+
}
|
|
772
|
+
/** Create the full-text index when the setup document enables one. */
|
|
773
|
+
async activateFulltextFromSetupDoc() {
|
|
774
|
+
try {
|
|
775
|
+
const config = await this.getFulltextSetup();
|
|
776
|
+
if (config?.enabled !== true ||
|
|
777
|
+
this.fulltextIndex ||
|
|
778
|
+
!this.fulltextAutoUpdateEnabled) {
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
// No explicit config: the index adopts the setup-document config on
|
|
782
|
+
// its first update run (seedConfigFromSetupDoc).
|
|
783
|
+
this.getFullTextIndex();
|
|
784
|
+
this.scheduleFulltextAutoUpdate();
|
|
785
|
+
}
|
|
786
|
+
catch (error) {
|
|
787
|
+
this.logger.warn(`Full-text auto-activation failed: ${error}`);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* Run a background full-text catch-up, coalescing further requests into
|
|
792
|
+
* one follow-up run (mirrors {@link scheduleSummaryAutoUpdate}).
|
|
793
|
+
*/
|
|
794
|
+
scheduleFulltextAutoUpdate() {
|
|
795
|
+
if (this.fulltextAutoUpdateRunning) {
|
|
796
|
+
this.fulltextAutoUpdatePending = true;
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
this.fulltextAutoUpdateRunning = true;
|
|
800
|
+
void (async () => {
|
|
801
|
+
try {
|
|
802
|
+
do {
|
|
803
|
+
this.fulltextAutoUpdatePending = false;
|
|
804
|
+
await this.fulltextIndex?.update();
|
|
805
|
+
} while (this.fulltextAutoUpdatePending && this.fulltextAutoUpdateEnabled);
|
|
806
|
+
}
|
|
807
|
+
catch (error) {
|
|
808
|
+
this.logger.warn(`Full-text auto-update failed: ${error}`);
|
|
809
|
+
}
|
|
810
|
+
finally {
|
|
811
|
+
this.fulltextAutoUpdateRunning = false;
|
|
812
|
+
}
|
|
813
|
+
})();
|
|
814
|
+
}
|
|
815
|
+
// ---------------------------------------------------------------------------
|
|
816
|
+
// Change notifications (reactive support)
|
|
817
|
+
// ---------------------------------------------------------------------------
|
|
818
|
+
/**
|
|
819
|
+
* Register a listener notified after documents change (local writes,
|
|
820
|
+
* sync ingest, access flips, witness updates). Events are coalesced:
|
|
821
|
+
* one event per sync batch, one per event-loop turn for local writes.
|
|
822
|
+
* The event carries only lightweight metadata plus the latest change
|
|
823
|
+
* cursor — no replay guarantee; consumers needing completeness read
|
|
824
|
+
* {@link iterateChangesSince} from their own cursor.
|
|
825
|
+
*
|
|
826
|
+
* @returns An unsubscribe function.
|
|
827
|
+
*/
|
|
828
|
+
addChangeListener(listener) {
|
|
829
|
+
this.changeListeners.add(listener);
|
|
830
|
+
return () => {
|
|
831
|
+
this.changeListeners.delete(listener);
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
/** Record a document transition for the coalesced change event. */
|
|
835
|
+
noteChangeForListeners(docId, lastModified, isDeleted) {
|
|
836
|
+
if (this.changeListeners.size === 0) {
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
839
|
+
this.pendingChangeNotifications.set(docId, { docId, isDeleted, lastModified });
|
|
840
|
+
this.scheduleChangeNotification();
|
|
841
|
+
}
|
|
842
|
+
scheduleChangeNotification() {
|
|
843
|
+
if (this.changeNotifyTimer !== null || this.changeNotificationHolds > 0) {
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
846
|
+
this.changeNotifyTimer = setTimeout(() => {
|
|
847
|
+
this.changeNotifyTimer = null;
|
|
848
|
+
this.emitPendingChangeEvent();
|
|
849
|
+
}, 0);
|
|
850
|
+
}
|
|
851
|
+
/**
|
|
852
|
+
* Suppress timer-based change emission for the duration of a batch
|
|
853
|
+
* operation (sync ingest); the release emits one coalesced event.
|
|
854
|
+
*/
|
|
855
|
+
beginChangeNotificationHold() {
|
|
856
|
+
this.changeNotificationHolds++;
|
|
857
|
+
}
|
|
858
|
+
endChangeNotificationHold() {
|
|
859
|
+
this.changeNotificationHolds = Math.max(0, this.changeNotificationHolds - 1);
|
|
860
|
+
if (this.changeNotificationHolds === 0) {
|
|
861
|
+
this.emitPendingChangeEvent();
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
emitPendingChangeEvent() {
|
|
865
|
+
if (this.changeNotificationHolds > 0) {
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
if (this.changeNotifyTimer !== null) {
|
|
869
|
+
clearTimeout(this.changeNotifyTimer);
|
|
870
|
+
this.changeNotifyTimer = null;
|
|
871
|
+
}
|
|
872
|
+
if (this.pendingChangeNotifications.size === 0) {
|
|
873
|
+
return;
|
|
874
|
+
}
|
|
875
|
+
const changes = Array.from(this.pendingChangeNotifications.values());
|
|
876
|
+
this.pendingChangeNotifications.clear();
|
|
877
|
+
if (this.changeListeners.size === 0) {
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
const event = {
|
|
881
|
+
changes,
|
|
882
|
+
cursor: this.getLatestChangeCursor(),
|
|
883
|
+
};
|
|
884
|
+
for (const listener of this.changeListeners) {
|
|
885
|
+
try {
|
|
886
|
+
listener(event);
|
|
887
|
+
}
|
|
888
|
+
catch (error) {
|
|
889
|
+
// Listener errors must never propagate into write/sync paths.
|
|
890
|
+
this.logger.warn(`Change listener threw: ${error}`);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Run an ad-hoc query against the document summary buffer (see
|
|
896
|
+
* docs/adhoc-queries.md). Filters and expression sort keys use the
|
|
897
|
+
* MindooDB expression language; documents are never materialized unless
|
|
898
|
+
* `options.allowFullScan` is set.
|
|
899
|
+
*/
|
|
900
|
+
async query(query, options) {
|
|
901
|
+
return (0, executeQuery_1.executeQuery)(this, this.getSummaryStore(), query, options);
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Build an ephemeral, summary-backed VirtualView (categories, sorting,
|
|
905
|
+
* totals as usual — but fed from the summary buffer, so no documents are
|
|
906
|
+
* materialized and re-sorting via `resort()` is a pure in-memory
|
|
907
|
+
* operation). The view is not cached; call `dispose()` when done.
|
|
908
|
+
*/
|
|
909
|
+
async queryView(definition, options) {
|
|
910
|
+
return (0, queryView_1.createEphemeralSummaryView)(this, this.getSummaryStore(), definition, options);
|
|
911
|
+
}
|
|
912
|
+
/**
|
|
913
|
+
* Live query: delivers the initial result asynchronously, then keeps
|
|
914
|
+
* re-evaluating after coalesced change events. `onResult` only fires
|
|
915
|
+
* when the result fingerprint (docIds + lastModified of the matches)
|
|
916
|
+
* actually changed.
|
|
917
|
+
*/
|
|
918
|
+
queryLive(query, onResult, options) {
|
|
919
|
+
return (0, queryLive_1.executeQueryLive)(this, this.getSummaryStore(), query, onResult, options);
|
|
279
920
|
}
|
|
280
921
|
getCachePrefix() {
|
|
281
922
|
return this.cachePrefix ?? `${this.tenant.getId()}/${this.store.getId()}`;
|
|
@@ -422,12 +1063,18 @@ class BaseMindooDB {
|
|
|
422
1063
|
* flush as well as from the upcoming flush-before-evict path.
|
|
423
1064
|
*/
|
|
424
1065
|
async flushDirtyDocToCache(store, prefix, docId, internal) {
|
|
425
|
-
const
|
|
426
|
-
const indexEntryIdx = this.indexLookup.get(docId);
|
|
1066
|
+
const indexEntryIdx = this.getDocIndexPosition(docId);
|
|
427
1067
|
const changeSeq = indexEntryIdx === undefined
|
|
428
1068
|
? 0
|
|
429
1069
|
: this.index[indexEntryIdx].changeSeq;
|
|
430
1070
|
const automergeHeads = automerge_adapter_1.Automerge.getHeads(internal.doc);
|
|
1071
|
+
// Skip the Automerge.save + L2 write when the exact same state was
|
|
1072
|
+
// already flushed (docs marked dirty by a load rather than a change).
|
|
1073
|
+
const flushKey = `${changeSeq}:${automergeHeads.join(",")}`;
|
|
1074
|
+
if (this.lastFlushedDocState.get(docId) === flushKey) {
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
const amBinary = automerge_adapter_1.Automerge.save(internal.doc);
|
|
431
1078
|
const header = JSON.stringify({
|
|
432
1079
|
version: DOC_CACHE_HEADER_VERSION,
|
|
433
1080
|
id: internal.id,
|
|
@@ -435,6 +1082,8 @@ class BaseMindooDB {
|
|
|
435
1082
|
lastModified: internal.lastModified,
|
|
436
1083
|
decryptionKeyId: internal.decryptionKeyId,
|
|
437
1084
|
isDeleted: internal.isDeleted,
|
|
1085
|
+
awaitingWitness: internal.awaitingWitness ?? false,
|
|
1086
|
+
witnessed: internal.witnessed ?? false,
|
|
438
1087
|
changeSeq,
|
|
439
1088
|
automergeHeads,
|
|
440
1089
|
});
|
|
@@ -446,6 +1095,7 @@ class BaseMindooDB {
|
|
|
446
1095
|
value.set(headerBytes, 4);
|
|
447
1096
|
value.set(amBinary, 4 + headerBytes.length);
|
|
448
1097
|
await store.put("doc", `${prefix}/${docId}`, value);
|
|
1098
|
+
this.lastFlushedDocState.set(docId, flushKey);
|
|
449
1099
|
}
|
|
450
1100
|
exportMetadataCheckpoint() {
|
|
451
1101
|
const checkpoint = {
|
|
@@ -501,6 +1151,7 @@ class BaseMindooDB {
|
|
|
501
1151
|
typeof checkpoint.keyBagFingerprint === "string" ? checkpoint.keyBagFingerprint : null;
|
|
502
1152
|
// Rebuild indexLookup from index
|
|
503
1153
|
this.indexLookup.clear();
|
|
1154
|
+
this.indexLookupStaleFrom = null;
|
|
504
1155
|
for (let i = 0; i < this.index.length; i++) {
|
|
505
1156
|
if (typeof this.index[i].changeSeq !== "number") {
|
|
506
1157
|
this.index[i].changeSeq = i + 1;
|
|
@@ -511,6 +1162,14 @@ class BaseMindooDB {
|
|
|
511
1162
|
if (!this.index[i].accessState) {
|
|
512
1163
|
this.index[i].accessState = "visible";
|
|
513
1164
|
}
|
|
1165
|
+
if (typeof this.index[i].awaitingWitness !== "boolean") {
|
|
1166
|
+
// Caches written before the witness-aware flag: treat as not awaiting.
|
|
1167
|
+
this.index[i].awaitingWitness = false;
|
|
1168
|
+
}
|
|
1169
|
+
if (typeof this.index[i].witnessed !== "boolean") {
|
|
1170
|
+
// Caches written before the witness-aware flag: treat as not witnessed.
|
|
1171
|
+
this.index[i].witnessed = false;
|
|
1172
|
+
}
|
|
514
1173
|
this.indexLookup.set(this.index[i].docId, i);
|
|
515
1174
|
}
|
|
516
1175
|
if (this.index.length > 0 && this.nextChangeSeq <= this.index.length) {
|
|
@@ -577,6 +1236,7 @@ class BaseMindooDB {
|
|
|
577
1236
|
const docId = batch[i].slice(docPrefix.length);
|
|
578
1237
|
await store.delete("doc", batch[i]);
|
|
579
1238
|
this.docCache.delete(docId);
|
|
1239
|
+
this.lastFlushedDocState.delete(docId);
|
|
580
1240
|
continue;
|
|
581
1241
|
}
|
|
582
1242
|
await this.storeCachedDocument(deserialized.internal);
|
|
@@ -629,6 +1289,8 @@ class BaseMindooDB {
|
|
|
629
1289
|
lastModified: header.lastModified,
|
|
630
1290
|
decryptionKeyId: header.decryptionKeyId,
|
|
631
1291
|
isDeleted: header.isDeleted,
|
|
1292
|
+
awaitingWitness: typeof header.awaitingWitness === "boolean" ? header.awaitingWitness : false,
|
|
1293
|
+
witnessed: typeof header.witnessed === "boolean" ? header.witnessed : false,
|
|
632
1294
|
};
|
|
633
1295
|
return {
|
|
634
1296
|
internal,
|
|
@@ -664,15 +1326,91 @@ class BaseMindooDB {
|
|
|
664
1326
|
return cryptoKey;
|
|
665
1327
|
}
|
|
666
1328
|
/**
|
|
667
|
-
*
|
|
668
|
-
*
|
|
1329
|
+
* Compute the author's metadata-binding signature for a new entry (audit
|
|
1330
|
+
* finding #5). The signature covers the canonical, versioned, length-prefixed
|
|
1331
|
+
* metadata layout (see crypto/EntrySignature.ts) rather than only the
|
|
1332
|
+
* ciphertext, so the cleartext metadata cannot be tampered with after signing.
|
|
1333
|
+
* Uses the supplied custom signing key when present, otherwise the current
|
|
1334
|
+
* user's signing key.
|
|
1335
|
+
*/
|
|
1336
|
+
async computeEntryMetadataSignature(meta, signing) {
|
|
1337
|
+
const bytes = (0, EntrySignature_1.buildEntrySigningBytes)((0, EntrySignature_1.entrySignatureFieldsFromEntry)(meta));
|
|
1338
|
+
if (signing?.signingKeyPair && signing?.signingKeyPassword) {
|
|
1339
|
+
return this.tenant.signPayloadWithKey(bytes, signing.signingKeyPair, signing.signingKeyPassword);
|
|
1340
|
+
}
|
|
1341
|
+
return this.tenant.signPayload(bytes);
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* Collect the full set of attachment references a document holds right now,
|
|
1345
|
+
* for the signed {@link StoreEntryMetadata.attachmentRefs} snapshot.
|
|
1346
|
+
*
|
|
1347
|
+
* Refs are sorted by `attachmentId` to give a canonical, deterministic order
|
|
1348
|
+
* so the bytes signed by the writer match the bytes a verifier reconstructs
|
|
1349
|
+
* (see crypto/EntrySignature.ts). Reads the merged `_attachments` array, so
|
|
1350
|
+
* additions, removals, and appends are all reflected.
|
|
669
1351
|
*/
|
|
670
|
-
|
|
1352
|
+
collectAttachmentRefs(doc) {
|
|
1353
|
+
const atts = doc._attachments ?? [];
|
|
1354
|
+
return atts
|
|
1355
|
+
.map((a) => ({
|
|
1356
|
+
attachmentId: a.attachmentId,
|
|
1357
|
+
lastChunkId: a.lastChunkId,
|
|
1358
|
+
size: a.size,
|
|
1359
|
+
}))
|
|
1360
|
+
.sort((x, y) => x.attachmentId < y.attachmentId ? -1 : x.attachmentId > y.attachmentId ? 1 : 0);
|
|
1361
|
+
}
|
|
1362
|
+
/**
|
|
1363
|
+
* Verify a store entry's author signature using a pre-imported CryptoKey,
|
|
1364
|
+
* version-aware (audit finding #5): prefers the metadata-binding
|
|
1365
|
+
* `metadataSignature` when present, otherwise the legacy ciphertext signature,
|
|
1366
|
+
* and re-verifies `SHA-256(encryptedData) === contentHash`. Returns false on
|
|
1367
|
+
* any mismatch. The caller is responsible for the directory trust check.
|
|
1368
|
+
*/
|
|
1369
|
+
async verifyEntrySignatureWithKey(cryptoKey, entry) {
|
|
671
1370
|
const subtle = this.tenant.getCryptoAdapter().getSubtle();
|
|
672
|
-
const
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
1371
|
+
const actualHash = await (0, idGeneration_1.computeContentHash)(entry.encryptedData, subtle);
|
|
1372
|
+
if (actualHash !== entry.contentHash) {
|
|
1373
|
+
this.logger.warn(`Content hash mismatch for entry ${entry.id}`);
|
|
1374
|
+
return false;
|
|
1375
|
+
}
|
|
1376
|
+
const requireMetadataSignature = await this.requiresMetadataSignature(entry);
|
|
1377
|
+
return (0, EntrySignature_1.verifyEntrySignatureWithImportedKey)(entry, entry.encryptedData, cryptoKey, subtle, {
|
|
1378
|
+
requireMetadataSignature,
|
|
1379
|
+
});
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* Whether the tenant storage-format floor (`requireMetadataSignatureSince`)
|
|
1383
|
+
* requires `entry` to carry the v2 metadata-binding signature, based on its
|
|
1384
|
+
* trusted time. The `directory` store is exempt (it must always load so the
|
|
1385
|
+
* policy that defines the floor can be read); entries that already carry a
|
|
1386
|
+
* `metadataSignature` short-circuit. Fail-open on resolution errors is safe —
|
|
1387
|
+
* the legacy signature is still cryptographically verified; the floor only
|
|
1388
|
+
* refuses the weaker fallback for new entries.
|
|
1389
|
+
*/
|
|
1390
|
+
async requiresMetadataSignature(entry) {
|
|
1391
|
+
if (entry.metadataSignature) {
|
|
1392
|
+
return false;
|
|
1393
|
+
}
|
|
1394
|
+
// Exempt the directory store from the floor: it must always materialize so
|
|
1395
|
+
// the policy can be read, and resolving the floor itself loads it.
|
|
1396
|
+
if (this.store.getId() === "directory") {
|
|
1397
|
+
return false;
|
|
1398
|
+
}
|
|
1399
|
+
try {
|
|
1400
|
+
const directory = await this.tenant.openDirectory();
|
|
1401
|
+
if (typeof directory.getRequireMetadataSignatureSince !== "function") {
|
|
1402
|
+
return false;
|
|
1403
|
+
}
|
|
1404
|
+
const cutoff = await directory.getRequireMetadataSignatureSince();
|
|
1405
|
+
if (cutoff === undefined) {
|
|
1406
|
+
return false;
|
|
1407
|
+
}
|
|
1408
|
+
return (0, storeEntryTime_1.entryTrustedTime)(entry, Date.now()) >= cutoff;
|
|
1409
|
+
}
|
|
1410
|
+
catch (error) {
|
|
1411
|
+
this.logger.warn(`Failed to resolve metadata-signature floor: ${String(error)}`);
|
|
1412
|
+
return false;
|
|
1413
|
+
}
|
|
676
1414
|
}
|
|
677
1415
|
/**
|
|
678
1416
|
* Compare two index entries for sorting.
|
|
@@ -706,7 +1444,7 @@ class BaseMindooDB {
|
|
|
706
1444
|
* @param accessState `"visible"` for readable docs, `"inaccessible"` for
|
|
707
1445
|
* tombstones produced by the key visibility layer
|
|
708
1446
|
*/
|
|
709
|
-
updateIndex(docId, lastModified, isDeleted, decryptionKeyId = "default", accessState = "visible") {
|
|
1447
|
+
updateIndex(docId, lastModified, isDeleted, decryptionKeyId = "default", accessState = "visible", awaitingWitness = false, witnessed = false) {
|
|
710
1448
|
const startedAt = Date.now();
|
|
711
1449
|
const assignedSeq = this.nextChangeSeq;
|
|
712
1450
|
const newEntry = {
|
|
@@ -716,26 +1454,28 @@ class BaseMindooDB {
|
|
|
716
1454
|
isDeleted,
|
|
717
1455
|
decryptionKeyId,
|
|
718
1456
|
accessState,
|
|
1457
|
+
awaitingWitness,
|
|
1458
|
+
witnessed,
|
|
719
1459
|
};
|
|
720
|
-
const existingIndex = this.
|
|
1460
|
+
const existingIndex = this.getDocIndexPosition(docId);
|
|
721
1461
|
// Check if the entry already exists and hasn't changed position
|
|
722
1462
|
if (existingIndex !== undefined) {
|
|
723
1463
|
const existingEntry = this.index[existingIndex];
|
|
724
1464
|
// If only metadata flags stayed identical and caller replays same state, skip.
|
|
1465
|
+
// `awaitingWitness` is part of this identity so a witness receipt (which
|
|
1466
|
+
// changes only `receivedAt`, not Automerge content) still bumps changeSeq
|
|
1467
|
+
// and re-emits the doc on the metadata feed.
|
|
725
1468
|
if (existingEntry.lastModified === lastModified
|
|
726
1469
|
&& existingEntry.isDeleted === isDeleted
|
|
727
1470
|
&& existingEntry.decryptionKeyId === decryptionKeyId
|
|
728
|
-
&& existingEntry.accessState === accessState
|
|
1471
|
+
&& existingEntry.accessState === accessState
|
|
1472
|
+
&& existingEntry.awaitingWitness === awaitingWitness
|
|
1473
|
+
&& existingEntry.witnessed === witnessed) {
|
|
729
1474
|
return; // No change needed
|
|
730
1475
|
}
|
|
731
|
-
// Remove from current position
|
|
1476
|
+
// Remove from current position; positions after it are repaired lazily
|
|
1477
|
+
// via indexLookupStaleFrom below.
|
|
732
1478
|
this.index.splice(existingIndex, 1);
|
|
733
|
-
// Update lookup map for entries that moved (only those after the removed position)
|
|
734
|
-
// We'll update the full range after insertion to be safe
|
|
735
|
-
const minAffectedIndex = Math.min(existingIndex, this.index.length);
|
|
736
|
-
for (let i = minAffectedIndex; i < this.index.length; i++) {
|
|
737
|
-
this.indexLookup.set(this.index[i].docId, i);
|
|
738
|
-
}
|
|
739
1479
|
this.indexLookup.delete(docId);
|
|
740
1480
|
}
|
|
741
1481
|
// Find insertion point using binary search to maintain sorted order
|
|
@@ -758,17 +1498,57 @@ class BaseMindooDB {
|
|
|
758
1498
|
// Insert at the correct position
|
|
759
1499
|
this.index.splice(insertIndex, 0, newEntry);
|
|
760
1500
|
this.nextChangeSeq = assignedSeq + 1;
|
|
761
|
-
|
|
762
|
-
//
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
1501
|
+
this.indexLookup.set(docId, insertIndex);
|
|
1502
|
+
// Entries at/after the first splice point may have shifted. Instead of
|
|
1503
|
+
// rewriting the lookup map here (O(index size) per update — twice, in the
|
|
1504
|
+
// old implementation), record the dirty range and let readers repair it
|
|
1505
|
+
// lazily. A fresh insert at the tail (the common bulk-sync case) shifts
|
|
1506
|
+
// nothing and stays clean.
|
|
1507
|
+
const shiftedFrom = existingIndex !== undefined
|
|
1508
|
+
? Math.min(existingIndex, insertIndex)
|
|
1509
|
+
: insertIndex;
|
|
1510
|
+
if (shiftedFrom < this.index.length - 1) {
|
|
1511
|
+
this.indexLookupStaleFrom = this.indexLookupStaleFrom === null
|
|
1512
|
+
? shiftedFrom
|
|
1513
|
+
: Math.min(this.indexLookupStaleFrom, shiftedFrom);
|
|
1514
|
+
}
|
|
1515
|
+
// Reactive support: record the transition for coalesced change events.
|
|
1516
|
+
// Reached only when the index actually changed (identical replays
|
|
1517
|
+
// returned early above).
|
|
1518
|
+
this.noteChangeForListeners(docId, lastModified, isDeleted);
|
|
766
1519
|
this.performanceCallback?.onIndexUpdate?.({
|
|
767
1520
|
docId,
|
|
768
1521
|
operation: existingIndex === undefined ? "insert" : "update",
|
|
769
1522
|
time: Date.now() - startedAt,
|
|
770
1523
|
});
|
|
771
1524
|
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Resolve a document's position in the sorted {@link index} through the
|
|
1527
|
+
* lookup map, repairing lazily-deferred staleness (see
|
|
1528
|
+
* {@link indexLookupStaleFrom}) only when the cached position no longer
|
|
1529
|
+
* matches. All `indexLookup.get` reads must go through this accessor.
|
|
1530
|
+
*/
|
|
1531
|
+
getDocIndexPosition(docId) {
|
|
1532
|
+
const pos = this.indexLookup.get(docId);
|
|
1533
|
+
if (pos === undefined) {
|
|
1534
|
+
return undefined;
|
|
1535
|
+
}
|
|
1536
|
+
if (this.index[pos]?.docId === docId) {
|
|
1537
|
+
return pos;
|
|
1538
|
+
}
|
|
1539
|
+
this.flushIndexLookupPatch();
|
|
1540
|
+
return this.indexLookup.get(docId);
|
|
1541
|
+
}
|
|
1542
|
+
/** Rewrite the stale tail of {@link indexLookup} recorded by {@link updateIndex}. */
|
|
1543
|
+
flushIndexLookupPatch() {
|
|
1544
|
+
if (this.indexLookupStaleFrom === null) {
|
|
1545
|
+
return;
|
|
1546
|
+
}
|
|
1547
|
+
for (let i = this.indexLookupStaleFrom; i < this.index.length; i++) {
|
|
1548
|
+
this.indexLookup.set(this.index[i].docId, i);
|
|
1549
|
+
}
|
|
1550
|
+
this.indexLookupStaleFrom = null;
|
|
1551
|
+
}
|
|
772
1552
|
/**
|
|
773
1553
|
* Register an automerge hash -> entry ID mapping for a document.
|
|
774
1554
|
* This is used to resolve Automerge dependency hashes to entry IDs.
|
|
@@ -940,6 +1720,7 @@ class BaseMindooDB {
|
|
|
940
1720
|
if (this.reconcileRestoredIndexOnInit) {
|
|
941
1721
|
await this.reconcileRestoredIndexWithStore();
|
|
942
1722
|
}
|
|
1723
|
+
this.probeFulltextSetupAtOpen();
|
|
943
1724
|
return;
|
|
944
1725
|
}
|
|
945
1726
|
}
|
|
@@ -948,6 +1729,29 @@ class BaseMindooDB {
|
|
|
948
1729
|
// docs whose keys are unavailable, so the only thing left to do is
|
|
949
1730
|
// record the current fingerprint for the next warm start.
|
|
950
1731
|
this.markKeyBagFingerprintReconciled(await this.computeCurrentKeyBagFingerprint());
|
|
1732
|
+
this.probeFulltextSetupAtOpen();
|
|
1733
|
+
}
|
|
1734
|
+
/**
|
|
1735
|
+
* Open-time probe for full-text auto-activation. Without it, a
|
|
1736
|
+
* database whose synced `fulltextSetup` is already enabled would not
|
|
1737
|
+
* build its index until the first change event arrives or the first
|
|
1738
|
+
* search runs. Fired once (fire-and-forget) at the end of
|
|
1739
|
+
* {@link initialize}; uses the same system-database exclusions and
|
|
1740
|
+
* activation path as the change-event probe (see
|
|
1741
|
+
* {@link handleChangesForFulltextAutoUpdate}).
|
|
1742
|
+
*/
|
|
1743
|
+
probeFulltextSetupAtOpen() {
|
|
1744
|
+
if (!this.fulltextAutoUpdateEnabled || this.fulltextSetupProbed || this.fulltextIndex) {
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
if (this._isAdminOnlyDb || this.store.getId() === "directory") {
|
|
1748
|
+
return;
|
|
1749
|
+
}
|
|
1750
|
+
if (this.isTimeTravelMode()) {
|
|
1751
|
+
return;
|
|
1752
|
+
}
|
|
1753
|
+
this.fulltextSetupProbed = true;
|
|
1754
|
+
void this.activateFulltextFromSetupDoc();
|
|
951
1755
|
}
|
|
952
1756
|
/**
|
|
953
1757
|
* Compute the current tenant KeyBag fingerprint, or `null` when the
|
|
@@ -1029,7 +1833,12 @@ class BaseMindooDB {
|
|
|
1029
1833
|
async purgeMaterializedDocument(docId) {
|
|
1030
1834
|
this.docCache.delete(docId);
|
|
1031
1835
|
this.dirtyDocIds.delete(docId);
|
|
1836
|
+
this.lastFlushedDocState.delete(docId);
|
|
1032
1837
|
this.automergeHashToEntryId.delete(docId);
|
|
1838
|
+
// Plaintext summary values must not outlive the purge either.
|
|
1839
|
+
this.summaryStore?.removeDocument(docId);
|
|
1840
|
+
// Nor extracted plaintext tokens in the full-text index.
|
|
1841
|
+
this.fulltextIndex?.removeDocument(docId);
|
|
1033
1842
|
const store = this.cacheManager?.getStore();
|
|
1034
1843
|
if (store) {
|
|
1035
1844
|
try {
|
|
@@ -1097,23 +1906,28 @@ class BaseMindooDB {
|
|
|
1097
1906
|
if (!visibility) {
|
|
1098
1907
|
continue;
|
|
1099
1908
|
}
|
|
1100
|
-
const existingIndex = this.
|
|
1909
|
+
const existingIndex = this.getDocIndexPosition(docId);
|
|
1101
1910
|
const existing = existingIndex === undefined ? undefined : this.index[existingIndex];
|
|
1911
|
+
// Visibility is governed by key possession alone: documents whose
|
|
1912
|
+
// decryption key is in the bag are visible; the rest are hidden. Which
|
|
1913
|
+
// keys a user holds is governed by the key distribution model
|
|
1914
|
+
// (docs/accesscontrol.md §13), not by a server/client read gate.
|
|
1102
1915
|
const canRead = await this.tenant.hasDecryptionKey(visibility.decryptionKeyId);
|
|
1103
1916
|
if (canRead) {
|
|
1104
1917
|
// Reveal-on-add only triggers when the previous state was
|
|
1105
1918
|
// missing or inaccessible; visible->visible transitions stay
|
|
1106
1919
|
// idempotent via the existing `updateIndex` short-circuit.
|
|
1107
1920
|
if (!existing || existing.accessState === "inaccessible") {
|
|
1108
|
-
|
|
1921
|
+
const { awaitingWitness, witnessed } = (0, storeEntryTime_1.metadataWitnessState)(metadataByDoc.get(docId));
|
|
1922
|
+
this.updateIndex(docId, visibility.lastModified, visibility.isDeleted, visibility.decryptionKeyId, "visible", awaitingWitness, witnessed);
|
|
1109
1923
|
changed = true;
|
|
1110
1924
|
}
|
|
1111
1925
|
continue;
|
|
1112
1926
|
}
|
|
1113
1927
|
if (existing?.accessState === "visible") {
|
|
1114
|
-
// Key was revoked while we still had plaintext state in cache:
|
|
1115
|
-
//
|
|
1116
|
-
//
|
|
1928
|
+
// Key was revoked while we still had plaintext state in cache: wipe it
|
|
1929
|
+
// and flip the index entry to an inaccessible tombstone with
|
|
1930
|
+
// `isDeleted: true` so view feeds emit a clean removal.
|
|
1117
1931
|
await this.purgeMaterializedDocument(docId);
|
|
1118
1932
|
this.updateIndex(docId, visibility.lastModified, true, visibility.decryptionKeyId, "inaccessible");
|
|
1119
1933
|
changed = true;
|
|
@@ -1152,7 +1966,9 @@ class BaseMindooDB {
|
|
|
1152
1966
|
`Rebuilding metadata from local store.`);
|
|
1153
1967
|
this.index = [];
|
|
1154
1968
|
this.indexLookup.clear();
|
|
1969
|
+
this.indexLookupStaleFrom = null;
|
|
1155
1970
|
this.docCache.clear();
|
|
1971
|
+
this.lastFlushedDocState.clear();
|
|
1156
1972
|
this.automergeHashToEntryId.clear();
|
|
1157
1973
|
this.processedEntryIds = [];
|
|
1158
1974
|
this.processedEntryCursor = null;
|
|
@@ -1175,6 +1991,17 @@ class BaseMindooDB {
|
|
|
1175
1991
|
* the workload is about to benefit from it.
|
|
1176
1992
|
*/
|
|
1177
1993
|
async syncStoreChanges() {
|
|
1994
|
+
// Hold change-notification emission for the whole batch so listeners
|
|
1995
|
+
// receive one coalesced event per sync run instead of one per document.
|
|
1996
|
+
this.beginChangeNotificationHold();
|
|
1997
|
+
try {
|
|
1998
|
+
await this.syncStoreChangesInternal();
|
|
1999
|
+
}
|
|
2000
|
+
finally {
|
|
2001
|
+
this.endChangeNotificationHold();
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
async syncStoreChangesInternal() {
|
|
1178
2005
|
const syncStartedAt = Date.now();
|
|
1179
2006
|
this.logger.debug(`Syncing store changes for database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
|
|
1180
2007
|
this.logger.debug(`Already processed ${this.processedEntryIds.length} entry IDs`);
|
|
@@ -1222,8 +2049,8 @@ class BaseMindooDB {
|
|
|
1222
2049
|
if (updatedDoc) {
|
|
1223
2050
|
this.logger.debug(`Successfully updated cached document ${docId} incrementally`);
|
|
1224
2051
|
// Only update index if document actually changed
|
|
1225
|
-
this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible");
|
|
1226
|
-
this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted})`);
|
|
2052
|
+
this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible", updatedDoc.awaitingWitness ?? false, updatedDoc.witnessed ?? false);
|
|
2053
|
+
this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted}, awaitingWitness: ${updatedDoc.awaitingWitness ?? false})`);
|
|
1227
2054
|
}
|
|
1228
2055
|
else {
|
|
1229
2056
|
this.logger.debug(`Document ${docId} unchanged after applying new entries, skipping index update`);
|
|
@@ -1235,7 +2062,7 @@ class BaseMindooDB {
|
|
|
1235
2062
|
this.docCache.delete(docId);
|
|
1236
2063
|
updatedDoc = await this.loadDocumentInternal(docId);
|
|
1237
2064
|
if (updatedDoc) {
|
|
1238
|
-
this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible");
|
|
2065
|
+
this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted, updatedDoc.decryptionKeyId, "visible", updatedDoc.awaitingWitness ?? false, updatedDoc.witnessed ?? false);
|
|
1239
2066
|
}
|
|
1240
2067
|
}
|
|
1241
2068
|
}
|
|
@@ -1268,8 +2095,9 @@ class BaseMindooDB {
|
|
|
1268
2095
|
const entriesForLastModified = mutationEntries.length > 0 ? mutationEntries : docLifecycleEntries;
|
|
1269
2096
|
const lastModified = Math.max(...entriesForLastModified.map((e) => e.createdAt));
|
|
1270
2097
|
const isDeleted = this.computeIsDeletedFromMetadata(allDocMetadata);
|
|
1271
|
-
|
|
1272
|
-
this.
|
|
2098
|
+
const { awaitingWitness, witnessed } = (0, storeEntryTime_1.metadataWitnessState)(allDocMetadata);
|
|
2099
|
+
this.updateIndex(docId, lastModified, isDeleted, representativeEntry.decryptionKeyId, "visible", awaitingWitness, witnessed);
|
|
2100
|
+
this.logger.debug(`Metadata-first update for uncached doc ${docId} (lastModified: ${lastModified}, isDeleted: ${isDeleted}, awaitingWitness: ${awaitingWitness})`);
|
|
1273
2101
|
}
|
|
1274
2102
|
}
|
|
1275
2103
|
}
|
|
@@ -1294,7 +2122,7 @@ class BaseMindooDB {
|
|
|
1294
2122
|
// existing index entry (if any). We do NOT need a second pass
|
|
1295
2123
|
// over the underlying store - the key is missing, so we just
|
|
1296
2124
|
// record an inaccessibility tombstone.
|
|
1297
|
-
const existingIndex = this.
|
|
2125
|
+
const existingIndex = this.getDocIndexPosition(docId);
|
|
1298
2126
|
const existing = existingIndex === undefined ? undefined : this.index[existingIndex];
|
|
1299
2127
|
if (existing?.accessState === "visible") {
|
|
1300
2128
|
const lifecycleEntriesInBatch = entryMetadataList.filter((entry) => entry.entryType === "doc_create"
|
|
@@ -1318,7 +2146,9 @@ class BaseMindooDB {
|
|
|
1318
2146
|
this.cacheMetaDirty = true;
|
|
1319
2147
|
this.cacheManager?.markDirty();
|
|
1320
2148
|
}
|
|
1321
|
-
this.
|
|
2149
|
+
if (!this.supportsCursorScan(this.store)) {
|
|
2150
|
+
this.processedEntryIds.push(...entryMetadataList.map(em => em.id));
|
|
2151
|
+
}
|
|
1322
2152
|
return;
|
|
1323
2153
|
}
|
|
1324
2154
|
this.logger.error(`===== ERROR processing document ${docId} in syncStoreChanges =====`, error);
|
|
@@ -1350,8 +2180,13 @@ class BaseMindooDB {
|
|
|
1350
2180
|
for (const [docId, entryMetadataList] of documentEntries) {
|
|
1351
2181
|
await processDocument(docId, entryMetadataList);
|
|
1352
2182
|
}
|
|
1353
|
-
// Append new entry IDs to our processed list
|
|
1354
|
-
|
|
2183
|
+
// Append new entry IDs to our processed list. Only needed for stores
|
|
2184
|
+
// without cursor scan (the checkpoint persists them only in that case);
|
|
2185
|
+
// cursor-capable stores track progress via processedEntryCursor, so we
|
|
2186
|
+
// avoid growing an unbounded in-memory id list.
|
|
2187
|
+
if (!this.supportsCursorScan(this.store)) {
|
|
2188
|
+
this.processedEntryIds.push(...newEntryMetadata.map(em => em.id));
|
|
2189
|
+
}
|
|
1355
2190
|
this.processedEntryCursor = nextCursor;
|
|
1356
2191
|
this.cacheMetaDirty = true;
|
|
1357
2192
|
this.cacheManager?.markDirty();
|
|
@@ -1563,7 +2398,14 @@ class BaseMindooDB {
|
|
|
1563
2398
|
if (signal?.aborted) {
|
|
1564
2399
|
return { transferred, cancelled: true };
|
|
1565
2400
|
}
|
|
1566
|
-
await targetStore.putEntries(batchEntries);
|
|
2401
|
+
const putResult = await targetStore.putEntries(batchEntries);
|
|
2402
|
+
// On push, the remote (witnessing) target returns receipts for accepted
|
|
2403
|
+
// entries; persist them back onto the local source so the revision feed
|
|
2404
|
+
// re-anchors them from the provisional head to their committed
|
|
2405
|
+
// `receivedAt` (docs/accesscontrol.md §5.3). Pull targets return void.
|
|
2406
|
+
if (Array.isArray(putResult) && putResult.length > 0 && sourceStore.applyWitnessReceipts) {
|
|
2407
|
+
await sourceStore.applyWitnessReceipts(putResult);
|
|
2408
|
+
}
|
|
1567
2409
|
transferred += batchEntries.length;
|
|
1568
2410
|
}
|
|
1569
2411
|
catch (error) {
|
|
@@ -1813,7 +2655,12 @@ class BaseMindooDB {
|
|
|
1813
2655
|
}
|
|
1814
2656
|
const batch = missingIds.slice(offset, offset + pageSize);
|
|
1815
2657
|
const entries = await sourceStore.getEntries(batch);
|
|
1816
|
-
await targetStore.putEntries(entries);
|
|
2658
|
+
const putResult = await targetStore.putEntries(entries);
|
|
2659
|
+
// Persist any witness receipts back onto the local source (see push path
|
|
2660
|
+
// in transferEntriesInBatches; docs/accesscontrol.md §5.3).
|
|
2661
|
+
if (Array.isArray(putResult) && putResult.length > 0 && sourceStore.applyWitnessReceipts) {
|
|
2662
|
+
await sourceStore.applyWitnessReceipts(putResult);
|
|
2663
|
+
}
|
|
1817
2664
|
transferred += entries.length;
|
|
1818
2665
|
onProgress?.({
|
|
1819
2666
|
phase: "transferring",
|
|
@@ -1894,6 +2741,105 @@ class BaseMindooDB {
|
|
|
1894
2741
|
async createDocumentWithSigningKey(signingKeyPair, signingKeyPassword, decryptionKeyId) {
|
|
1895
2742
|
return this.createDocumentInternal({ signingKeyPair, signingKeyPassword, decryptionKeyId });
|
|
1896
2743
|
}
|
|
2744
|
+
/**
|
|
2745
|
+
* Bulk-create documents in a single append-only store write.
|
|
2746
|
+
*
|
|
2747
|
+
* Semantically equivalent to calling {@link createDocument} once per input,
|
|
2748
|
+
* with these bulk-oriented differences:
|
|
2749
|
+
* - All produced store entries are written through ONE `putEntries()` call,
|
|
2750
|
+
* so IndexedDB-backed stores commit the whole batch in a single readwrite
|
|
2751
|
+
* transaction instead of one per document.
|
|
2752
|
+
* - Custom-id documents MAY combine `id` with `initialValues`: the values are
|
|
2753
|
+
* applied as a separate `doc_change` entry on top of the deterministic
|
|
2754
|
+
* seed change, and both entries land in the same batch. (The single-create
|
|
2755
|
+
* path rejects this combination and requires a follow-up `changeDoc()`.)
|
|
2756
|
+
* - Freshly created documents are cached but intentionally NOT added to the
|
|
2757
|
+
* dirty-doc tracking: their entries are already durable in the store, so
|
|
2758
|
+
* an L1 eviction can always re-materialize them. This avoids the per-doc
|
|
2759
|
+
* flush-before-evict L2 write storm during large imports.
|
|
2760
|
+
* - Snapshot scheduling is skipped (fresh documents cannot exceed the
|
|
2761
|
+
* snapshot threshold).
|
|
2762
|
+
*
|
|
2763
|
+
* Existing custom-id documents keep the idempotent single-create semantics:
|
|
2764
|
+
* the existing document is returned (tombstones are undeleted first) and any
|
|
2765
|
+
* `initialValues` are applied as a follow-up change.
|
|
2766
|
+
*
|
|
2767
|
+
* Fail-fast: invalid inputs are rejected up front, and a denied ACL
|
|
2768
|
+
* precheck rejects the whole call before any fresh document is written.
|
|
2769
|
+
*/
|
|
2770
|
+
async createDocuments(inputs) {
|
|
2771
|
+
this.assertWritable("createDocuments");
|
|
2772
|
+
if (inputs.length === 0) {
|
|
2773
|
+
return [];
|
|
2774
|
+
}
|
|
2775
|
+
this.logger.info(`Bulk-creating ${inputs.length} documents`);
|
|
2776
|
+
// Fail-fast input validation BEFORE any write: an invalid input anywhere
|
|
2777
|
+
// in the batch must reject the whole call without side effects.
|
|
2778
|
+
for (const input of inputs) {
|
|
2779
|
+
const options = input ?? {};
|
|
2780
|
+
if ((options.signingKeyPair !== undefined) !== (options.signingKeyPassword !== undefined)) {
|
|
2781
|
+
throw new Error("createDocuments: signingKeyPair and signingKeyPassword must be provided together");
|
|
2782
|
+
}
|
|
2783
|
+
if (options.id !== undefined && !types_1.CUSTOM_DOC_ID_REGEX.test(options.id)) {
|
|
2784
|
+
throw new Error(`createDocuments: invalid document id "${options.id}". ` +
|
|
2785
|
+
`Custom document IDs must match ${types_1.CUSTOM_DOC_ID_REGEX.source}.`);
|
|
2786
|
+
}
|
|
2787
|
+
}
|
|
2788
|
+
const results = new Array(inputs.length).fill(null);
|
|
2789
|
+
const prepared = [];
|
|
2790
|
+
for (let index = 0; index < inputs.length; index++) {
|
|
2791
|
+
const options = inputs[index] ?? {};
|
|
2792
|
+
if (options.id !== undefined) {
|
|
2793
|
+
// Metadata-only existence probe via the store's docId index (cheap even
|
|
2794
|
+
// on IndexedDB); avoids a full materialization for the common
|
|
2795
|
+
// fresh-document case.
|
|
2796
|
+
const existingMetadata = await this.store.findNewEntriesForDoc([], options.id);
|
|
2797
|
+
if (existingMetadata.length > 0) {
|
|
2798
|
+
// Idempotent path (rare during bulk imports): reuse the single-create
|
|
2799
|
+
// semantics (returns the existing doc, undeletes tombstones) and
|
|
2800
|
+
// apply initialValues as a follow-up change, mirroring the non-bulk
|
|
2801
|
+
// create + changeDoc flow.
|
|
2802
|
+
const { initialValues, ...rest } = options;
|
|
2803
|
+
const existingDoc = await this.createDocumentInternal(rest);
|
|
2804
|
+
const valueEntries = initialValues
|
|
2805
|
+
? Object.entries(initialValues).filter(([k, v]) => !k.startsWith("_") && v !== undefined)
|
|
2806
|
+
: [];
|
|
2807
|
+
if (valueEntries.length > 0) {
|
|
2808
|
+
await this.changeDocInternal(existingDoc, (mutable) => {
|
|
2809
|
+
const data = mutable.getData();
|
|
2810
|
+
for (const [key, value] of valueEntries) {
|
|
2811
|
+
data[key] = value;
|
|
2812
|
+
}
|
|
2813
|
+
}, options.signingKeyPair, options.signingKeyPassword, options.bypassAccessControlPrecheck);
|
|
2814
|
+
}
|
|
2815
|
+
results[index] = existingDoc;
|
|
2816
|
+
continue;
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
const preparedDoc = await this.prepareFreshDocumentCreate(options);
|
|
2820
|
+
prepared.push({ index, ...preparedDoc });
|
|
2821
|
+
}
|
|
2822
|
+
const allEntries = [];
|
|
2823
|
+
for (const preparedDoc of prepared) {
|
|
2824
|
+
allEntries.push(...preparedDoc.entries);
|
|
2825
|
+
}
|
|
2826
|
+
if (allEntries.length > 0) {
|
|
2827
|
+
await this.store.putEntries(allEntries);
|
|
2828
|
+
}
|
|
2829
|
+
for (const { index, internalDoc } of prepared) {
|
|
2830
|
+
await this.storeCachedDocument(internalDoc);
|
|
2831
|
+
this.updateIndex(internalDoc.id, internalDoc.lastModified, false, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness ?? false, internalDoc.witnessed ?? false);
|
|
2832
|
+
results[index] = this.wrapDocument(internalDoc);
|
|
2833
|
+
}
|
|
2834
|
+
if (prepared.length > 0) {
|
|
2835
|
+
// One metadata-checkpoint invalidation for the whole batch. The created
|
|
2836
|
+
// docs are intentionally NOT added to dirtyDocIds (see method docs).
|
|
2837
|
+
this.cacheMetaDirty = true;
|
|
2838
|
+
this.cacheManager?.markDirty();
|
|
2839
|
+
}
|
|
2840
|
+
this.logger.info(`Bulk-created ${prepared.length} documents (${inputs.length - prepared.length} already existed)`);
|
|
2841
|
+
return results;
|
|
2842
|
+
}
|
|
1897
2843
|
/**
|
|
1898
2844
|
* Internal method to create a new document.
|
|
1899
2845
|
*
|
|
@@ -1914,13 +2860,30 @@ class BaseMindooDB {
|
|
|
1914
2860
|
throw new Error("createDocument: signingKeyPair and signingKeyPassword must be provided together");
|
|
1915
2861
|
}
|
|
1916
2862
|
const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
|
|
1917
|
-
const keyId = options
|
|
2863
|
+
const keyId = await this.resolveCreateKeyId(options);
|
|
1918
2864
|
const useCustomDocId = options.id !== undefined;
|
|
2865
|
+
// Sanitize caller-provided initial values: internal/reserved fields (those
|
|
2866
|
+
// starting with "_", e.g. "_attachments") are managed by MindooDB and must
|
|
2867
|
+
// not be seeded by callers. `undefined` values are treated as "not set"
|
|
2868
|
+
// (Automerge cannot represent undefined).
|
|
2869
|
+
const initialValueEntries = options.initialValues
|
|
2870
|
+
? Object.entries(options.initialValues).filter(([k, v]) => !k.startsWith("_") && v !== undefined)
|
|
2871
|
+
: [];
|
|
2872
|
+
const hasInitialValues = initialValueEntries.length > 0;
|
|
1919
2873
|
if (useCustomDocId) {
|
|
1920
2874
|
if (!types_1.CUSTOM_DOC_ID_REGEX.test(options.id)) {
|
|
1921
2875
|
throw new Error(`createDocument: invalid document id "${options.id}". ` +
|
|
1922
2876
|
`Custom document IDs must match ${types_1.CUSTOM_DOC_ID_REGEX.source}.`);
|
|
1923
2877
|
}
|
|
2878
|
+
// Custom-ID documents are seeded with a hard-coded initial change so that
|
|
2879
|
+
// independent replicas converge on the same Automerge hash. Baking
|
|
2880
|
+
// caller values into that change would diverge the hash per content, so
|
|
2881
|
+
// initialValues is unsupported on the custom-ID path in v1. Create the
|
|
2882
|
+
// document first, then apply values via changeDoc().
|
|
2883
|
+
if (hasInitialValues) {
|
|
2884
|
+
throw new Error("createDocument: initialValues is not supported together with a custom id; " +
|
|
2885
|
+
"create the document, then apply values with changeDoc().");
|
|
2886
|
+
}
|
|
1924
2887
|
}
|
|
1925
2888
|
// Admin-only validation: only admin key can modify data in admin-only databases
|
|
1926
2889
|
if (this._isAdminOnlyDb) {
|
|
@@ -1968,7 +2931,8 @@ class BaseMindooDB {
|
|
|
1968
2931
|
const fresh = automerge_adapter_1.Automerge.init();
|
|
1969
2932
|
const [appliedDoc] = automerge_adapter_1.Automerge.applyChanges(fresh, [changeBytes]);
|
|
1970
2933
|
newDoc = appliedDoc;
|
|
1971
|
-
|
|
2934
|
+
if (this.isDebugEnabled())
|
|
2935
|
+
this.logger.debug(`Applied hard-coded initial change, heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
1972
2936
|
}
|
|
1973
2937
|
catch (error) {
|
|
1974
2938
|
this.logger.error(`Error applying hard-coded initial change for document ${docId}:`, error);
|
|
@@ -1983,8 +2947,15 @@ class BaseMindooDB {
|
|
|
1983
2947
|
// Store metadata in the document payload
|
|
1984
2948
|
// We need to modify the document to ensure a change is created
|
|
1985
2949
|
doc._attachments = [];
|
|
2950
|
+
// Seed caller-provided initial values within the same doc_create
|
|
2951
|
+
// change so a doc_create Tier 2 rule can evaluate them in the
|
|
2952
|
+
// "after" state (docs/accesscontrol.md §6.3, §9).
|
|
2953
|
+
for (const [key, value] of initialValueEntries) {
|
|
2954
|
+
doc[key] = value;
|
|
2955
|
+
}
|
|
1986
2956
|
});
|
|
1987
|
-
|
|
2957
|
+
if (this.isDebugEnabled())
|
|
2958
|
+
this.logger.debug(`Successfully created Automerge change, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
1988
2959
|
}
|
|
1989
2960
|
catch (error) {
|
|
1990
2961
|
this.logger.error(`Error in Automerge.change for document ${docId}:`, error);
|
|
@@ -1998,6 +2969,23 @@ class BaseMindooDB {
|
|
|
1998
2969
|
changeBytes = localChange;
|
|
1999
2970
|
}
|
|
2000
2971
|
this.logger.debug(`Got change bytes: ${changeBytes.length} bytes`);
|
|
2972
|
+
// Client-side write-policy precheck (docs/accesscontrol.md §9). Evaluates
|
|
2973
|
+
// the full Tier 1 + Tier 2 ruleset for this doc_create and throws
|
|
2974
|
+
// AccessDeniedError when denied, before we spend work encrypting/signing and
|
|
2975
|
+
// writing. The server witness remains authoritative; this only gives the
|
|
2976
|
+
// caller an immediate, meaningful error.
|
|
2977
|
+
{
|
|
2978
|
+
const signerKey = useCustomSigningKey
|
|
2979
|
+
? signingKeyPair.publicKey
|
|
2980
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
2981
|
+
await this.assertWriteAllowed({
|
|
2982
|
+
op: "doc_create",
|
|
2983
|
+
signerKey,
|
|
2984
|
+
isAuthor: true,
|
|
2985
|
+
bypass: options.bypassAccessControlPrecheck,
|
|
2986
|
+
getAfterDoc: () => this.convertAutomergeToJS(newDoc),
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2001
2989
|
// Decode the change to get hash and dependencies
|
|
2002
2990
|
this.logger.debug(`Decoding change to get hash and dependencies`);
|
|
2003
2991
|
let decodedChange;
|
|
@@ -2038,6 +3026,12 @@ class BaseMindooDB {
|
|
|
2038
3026
|
createdByPublicKey = currentUser.userSigningPublicKey;
|
|
2039
3027
|
}
|
|
2040
3028
|
this.logger.debug(`Signed payload, signature length: ${signature.length} bytes`);
|
|
3029
|
+
// Remember the creator's signing key so $author prechecks on subsequent
|
|
3030
|
+
// change/delete operations don't need a metadata scan.
|
|
3031
|
+
this.creatorKeyCache.set(docId, createdByPublicKey);
|
|
3032
|
+
// Snapshot of attachments referenced by the freshly created doc (normally
|
|
3033
|
+
// empty; the public create API does not add attachments inline).
|
|
3034
|
+
const createAttachmentRefs = this.collectAttachmentRefs(newDoc);
|
|
2041
3035
|
// Create entry metadata
|
|
2042
3036
|
const entryMetadata = {
|
|
2043
3037
|
entryType: "doc_create",
|
|
@@ -2051,7 +3045,11 @@ class BaseMindooDB {
|
|
|
2051
3045
|
signature,
|
|
2052
3046
|
originalSize: changeBytes.length,
|
|
2053
3047
|
encryptedSize: encryptedPayload.length,
|
|
3048
|
+
attachmentRefs: createAttachmentRefs.length > 0 ? createAttachmentRefs : undefined,
|
|
3049
|
+
entryVersion: types_1.CURRENT_STORE_ENTRY_VERSION,
|
|
2054
3050
|
};
|
|
3051
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
3052
|
+
entryMetadata.metadataSignature = await this.computeEntryMetadataSignature(entryMetadata, useCustomSigningKey ? { signingKeyPair, signingKeyPassword } : undefined);
|
|
2055
3053
|
// Create full entry object
|
|
2056
3054
|
const fullEntry = {
|
|
2057
3055
|
...entryMetadata,
|
|
@@ -2061,6 +3059,12 @@ class BaseMindooDB {
|
|
|
2061
3059
|
await this.store.putEntries([fullEntry]);
|
|
2062
3060
|
// Register automerge hash -> entry ID mapping
|
|
2063
3061
|
this.registerAutomergeHashMapping(docId, automergeHash, entryId);
|
|
3062
|
+
// A freshly created entry is versioned and un-witnessed, so the document is
|
|
3063
|
+
// awaiting witness until it is pushed and stamped with a `receivedAt`.
|
|
3064
|
+
const awaitingWitness = (0, storeEntryTime_1.isProvisional)(entryMetadata);
|
|
3065
|
+
// Witnessed only once a versioned entry carries a `receivedAt`; a brand-new
|
|
3066
|
+
// local create is versioned-but-provisional, so it is not yet witnessed.
|
|
3067
|
+
const witnessed = (0, storeEntryTime_1.isVersioned)(entryMetadata) && !awaitingWitness;
|
|
2064
3068
|
// Create internal document representation
|
|
2065
3069
|
const internalDoc = {
|
|
2066
3070
|
id: docId,
|
|
@@ -2069,22 +3073,205 @@ class BaseMindooDB {
|
|
|
2069
3073
|
lastModified: now,
|
|
2070
3074
|
decryptionKeyId: keyId,
|
|
2071
3075
|
isDeleted: false,
|
|
3076
|
+
awaitingWitness,
|
|
3077
|
+
witnessed,
|
|
2072
3078
|
};
|
|
2073
3079
|
// Update cache and index
|
|
2074
3080
|
await this.storeCachedDocument(internalDoc);
|
|
2075
|
-
this.updateIndex(docId, internalDoc.lastModified, false, internalDoc.decryptionKeyId, "visible");
|
|
3081
|
+
this.updateIndex(docId, internalDoc.lastModified, false, internalDoc.decryptionKeyId, "visible", awaitingWitness, witnessed);
|
|
2076
3082
|
this.markDocDirty(docId);
|
|
2077
3083
|
this.logger.info(`Document ${docId} created successfully`);
|
|
2078
3084
|
this.logger.debug(`Document ${docId} cached and indexed (lastModified: ${internalDoc.lastModified})`);
|
|
2079
3085
|
return this.wrapDocument(internalDoc);
|
|
2080
3086
|
}
|
|
3087
|
+
/**
|
|
3088
|
+
* Build the store entries and in-memory state for one fresh document of a
|
|
3089
|
+
* {@link createDocuments} batch WITHOUT writing anything. The caller is
|
|
3090
|
+
* responsible for persisting the returned entries (batched `putEntries`)
|
|
3091
|
+
* and for cache/index registration afterwards.
|
|
3092
|
+
*
|
|
3093
|
+
* Mirrors {@link createDocumentInternal} for the fresh-document case, with
|
|
3094
|
+
* one bulk-only extension: a custom-id document MAY carry `initialValues`,
|
|
3095
|
+
* which are emitted as an extra `doc_change` entry on top of the
|
|
3096
|
+
* deterministic seed change so both land in the same store batch.
|
|
3097
|
+
*/
|
|
3098
|
+
async prepareFreshDocumentCreate(options) {
|
|
3099
|
+
const { signingKeyPair, signingKeyPassword } = options;
|
|
3100
|
+
if ((signingKeyPair !== undefined) !== (signingKeyPassword !== undefined)) {
|
|
3101
|
+
throw new Error("createDocuments: signingKeyPair and signingKeyPassword must be provided together");
|
|
3102
|
+
}
|
|
3103
|
+
const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
|
|
3104
|
+
const keyId = await this.resolveCreateKeyId(options);
|
|
3105
|
+
const useCustomDocId = options.id !== undefined;
|
|
3106
|
+
// Sanitize caller-provided initial values: internal/reserved fields are
|
|
3107
|
+
// managed by MindooDB and must not be seeded by callers. `undefined`
|
|
3108
|
+
// values are treated as "not set" (Automerge cannot represent undefined).
|
|
3109
|
+
const initialValueEntries = options.initialValues
|
|
3110
|
+
? Object.entries(options.initialValues).filter(([k, v]) => !k.startsWith("_") && v !== undefined)
|
|
3111
|
+
: [];
|
|
3112
|
+
// Admin-only validation: only admin key can modify data in admin-only databases
|
|
3113
|
+
if (this._isAdminOnlyDb) {
|
|
3114
|
+
const adminPublicKey = this.getAdminPublicKey();
|
|
3115
|
+
const signerPublicKey = useCustomSigningKey
|
|
3116
|
+
? signingKeyPair.publicKey
|
|
3117
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
3118
|
+
if (signerPublicKey !== adminPublicKey) {
|
|
3119
|
+
throw new Error("Admin-only database: only the admin key can modify data");
|
|
3120
|
+
}
|
|
3121
|
+
}
|
|
3122
|
+
const docId = useCustomDocId ? options.id : (0, uuid_1.v7)();
|
|
3123
|
+
const now = Date.now();
|
|
3124
|
+
// Build the initial Automerge document and its first change bytes,
|
|
3125
|
+
// following the same seeding rules as the single-create path.
|
|
3126
|
+
let seedDoc;
|
|
3127
|
+
let createChangeBytes;
|
|
3128
|
+
if (useCustomDocId) {
|
|
3129
|
+
createChangeBytes = getCustomIdInitialChangeBytes();
|
|
3130
|
+
const fresh = automerge_adapter_1.Automerge.init();
|
|
3131
|
+
const [appliedDoc] = automerge_adapter_1.Automerge.applyChanges(fresh, [createChangeBytes]);
|
|
3132
|
+
seedDoc = appliedDoc;
|
|
3133
|
+
}
|
|
3134
|
+
else {
|
|
3135
|
+
const initialDoc = automerge_adapter_1.Automerge.init();
|
|
3136
|
+
seedDoc = automerge_adapter_1.Automerge.change(initialDoc, (doc) => {
|
|
3137
|
+
doc._attachments = [];
|
|
3138
|
+
// Seed caller-provided initial values within the same doc_create change
|
|
3139
|
+
// so a doc_create Tier 2 rule can evaluate them in the "after" state.
|
|
3140
|
+
for (const [key, value] of initialValueEntries) {
|
|
3141
|
+
doc[key] = value;
|
|
3142
|
+
}
|
|
3143
|
+
});
|
|
3144
|
+
const localChange = automerge_adapter_1.Automerge.getLastLocalChange(seedDoc);
|
|
3145
|
+
if (!localChange) {
|
|
3146
|
+
throw new Error("Failed to get change bytes from Automerge document");
|
|
3147
|
+
}
|
|
3148
|
+
createChangeBytes = localChange;
|
|
3149
|
+
}
|
|
3150
|
+
// Custom-id + initialValues (bulk-only): the deterministic seed change must
|
|
3151
|
+
// stay byte-identical across replicas, so the values go into a separate
|
|
3152
|
+
// follow-up change that ships in the same batch.
|
|
3153
|
+
let finalDoc = seedDoc;
|
|
3154
|
+
let valueChangeBytes = null;
|
|
3155
|
+
if (useCustomDocId && initialValueEntries.length > 0) {
|
|
3156
|
+
finalDoc = automerge_adapter_1.Automerge.change(seedDoc, { time: now }, (doc) => {
|
|
3157
|
+
for (const [key, value] of initialValueEntries) {
|
|
3158
|
+
doc[key] = value;
|
|
3159
|
+
}
|
|
3160
|
+
});
|
|
3161
|
+
const localChange = automerge_adapter_1.Automerge.getLastLocalChange(finalDoc);
|
|
3162
|
+
if (!localChange) {
|
|
3163
|
+
throw new Error("Failed to get change bytes from Automerge document");
|
|
3164
|
+
}
|
|
3165
|
+
valueChangeBytes = localChange;
|
|
3166
|
+
}
|
|
3167
|
+
// Client-side write-policy precheck (docs/accesscontrol.md §9) against the
|
|
3168
|
+
// document's final after-state (seed + initial values), so a doc_create
|
|
3169
|
+
// Tier 2 rule sees the same state it would see on the single-create path.
|
|
3170
|
+
{
|
|
3171
|
+
const signerKey = useCustomSigningKey
|
|
3172
|
+
? signingKeyPair.publicKey
|
|
3173
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
3174
|
+
const afterDoc = finalDoc;
|
|
3175
|
+
await this.assertWriteAllowed({
|
|
3176
|
+
op: "doc_create",
|
|
3177
|
+
signerKey,
|
|
3178
|
+
isAuthor: true,
|
|
3179
|
+
bypass: options.bypassAccessControlPrecheck,
|
|
3180
|
+
getAfterDoc: () => this.convertAutomergeToJS(afterDoc),
|
|
3181
|
+
});
|
|
3182
|
+
}
|
|
3183
|
+
// Build the doc_create entry.
|
|
3184
|
+
const decodedCreate = automerge_adapter_1.Automerge.decodeChange(createChangeBytes);
|
|
3185
|
+
const createHash = decodedCreate.hash;
|
|
3186
|
+
const createDepHashes = decodedCreate.deps || [];
|
|
3187
|
+
const encryptedPayload = await this.tenant.encryptPayload(createChangeBytes, keyId);
|
|
3188
|
+
const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedPayload, this.getSubtle());
|
|
3189
|
+
const entryId = await (0, idGeneration_1.generateDocEntryId)(docId, createHash, createDepHashes, this.getSubtle());
|
|
3190
|
+
const dependencyIds = await this.ensureAutomergeDepsResolved(docId, createDepHashes);
|
|
3191
|
+
let signature;
|
|
3192
|
+
let createdByPublicKey;
|
|
3193
|
+
if (useCustomSigningKey) {
|
|
3194
|
+
signature = await this.tenant.signPayloadWithKey(encryptedPayload, signingKeyPair, signingKeyPassword);
|
|
3195
|
+
createdByPublicKey = signingKeyPair.publicKey;
|
|
3196
|
+
}
|
|
3197
|
+
else {
|
|
3198
|
+
const currentUser = await this.tenant.getCurrentUserId();
|
|
3199
|
+
signature = await this.tenant.signPayload(encryptedPayload);
|
|
3200
|
+
createdByPublicKey = currentUser.userSigningPublicKey;
|
|
3201
|
+
}
|
|
3202
|
+
// Remember the creator's signing key so $author prechecks on subsequent
|
|
3203
|
+
// change/delete operations don't need a metadata scan.
|
|
3204
|
+
this.creatorKeyCache.set(docId, createdByPublicKey);
|
|
3205
|
+
const createAttachmentRefs = this.collectAttachmentRefs(finalDoc);
|
|
3206
|
+
const entryMetadata = {
|
|
3207
|
+
entryType: "doc_create",
|
|
3208
|
+
id: entryId,
|
|
3209
|
+
contentHash,
|
|
3210
|
+
docId,
|
|
3211
|
+
dependencyIds,
|
|
3212
|
+
createdAt: now,
|
|
3213
|
+
createdByPublicKey,
|
|
3214
|
+
decryptionKeyId: keyId,
|
|
3215
|
+
signature,
|
|
3216
|
+
originalSize: createChangeBytes.length,
|
|
3217
|
+
encryptedSize: encryptedPayload.length,
|
|
3218
|
+
attachmentRefs: createAttachmentRefs.length > 0 ? createAttachmentRefs : undefined,
|
|
3219
|
+
entryVersion: types_1.CURRENT_STORE_ENTRY_VERSION,
|
|
3220
|
+
};
|
|
3221
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
3222
|
+
entryMetadata.metadataSignature = await this.computeEntryMetadataSignature(entryMetadata, useCustomSigningKey ? { signingKeyPair, signingKeyPassword } : undefined);
|
|
3223
|
+
const entries = [{
|
|
3224
|
+
...entryMetadata,
|
|
3225
|
+
encryptedData: encryptedPayload,
|
|
3226
|
+
}];
|
|
3227
|
+
// Register the mapping BEFORE building the follow-up change entry so its
|
|
3228
|
+
// dependency resolution finds the doc_create parent.
|
|
3229
|
+
this.registerAutomergeHashMapping(docId, createHash, entryId);
|
|
3230
|
+
if (valueChangeBytes) {
|
|
3231
|
+
const tempInternalDoc = {
|
|
3232
|
+
id: docId,
|
|
3233
|
+
doc: finalDoc,
|
|
3234
|
+
createdAt: now,
|
|
3235
|
+
lastModified: now,
|
|
3236
|
+
decryptionKeyId: keyId,
|
|
3237
|
+
isDeleted: false,
|
|
3238
|
+
};
|
|
3239
|
+
const changeEntry = await this.buildDocChangeStoreEntry({
|
|
3240
|
+
internalDoc: tempInternalDoc,
|
|
3241
|
+
changeBytes: valueChangeBytes,
|
|
3242
|
+
now,
|
|
3243
|
+
useCustomKey: useCustomSigningKey,
|
|
3244
|
+
signingKeyPair,
|
|
3245
|
+
signingKeyPassword,
|
|
3246
|
+
attachmentRefs: createAttachmentRefs,
|
|
3247
|
+
});
|
|
3248
|
+
entries.push(changeEntry);
|
|
3249
|
+
const decodedValueChange = automerge_adapter_1.Automerge.decodeChange(valueChangeBytes);
|
|
3250
|
+
this.registerAutomergeHashMapping(docId, decodedValueChange.hash, changeEntry.id);
|
|
3251
|
+
}
|
|
3252
|
+
// Freshly created entries are versioned and un-witnessed, so the document
|
|
3253
|
+
// is awaiting witness until pushed and stamped with a receipt.
|
|
3254
|
+
const awaitingWitness = entries.some(storeEntryTime_1.isProvisional);
|
|
3255
|
+
const witnessed = entries.some(storeEntryTime_1.isVersioned) && !awaitingWitness;
|
|
3256
|
+
const internalDoc = {
|
|
3257
|
+
id: docId,
|
|
3258
|
+
doc: finalDoc,
|
|
3259
|
+
createdAt: now,
|
|
3260
|
+
lastModified: now,
|
|
3261
|
+
decryptionKeyId: keyId,
|
|
3262
|
+
isDeleted: false,
|
|
3263
|
+
awaitingWitness,
|
|
3264
|
+
witnessed,
|
|
3265
|
+
};
|
|
3266
|
+
return { internalDoc, entries };
|
|
3267
|
+
}
|
|
2081
3268
|
async getDocument(docId) {
|
|
2082
3269
|
const internalDoc = await this.loadDocumentInternal(docId);
|
|
2083
3270
|
if (!internalDoc) {
|
|
2084
|
-
throw new
|
|
3271
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
2085
3272
|
}
|
|
2086
3273
|
if (internalDoc.isDeleted) {
|
|
2087
|
-
throw new
|
|
3274
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
2088
3275
|
}
|
|
2089
3276
|
return this.wrapDocument(internalDoc);
|
|
2090
3277
|
}
|
|
@@ -2134,7 +3321,7 @@ class BaseMindooDB {
|
|
|
2134
3321
|
this.logger.warn(`Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
|
|
2135
3322
|
}
|
|
2136
3323
|
else {
|
|
2137
|
-
isValid = await this.tenant.
|
|
3324
|
+
isValid = await this.tenant.verifyEntrySignature(snapshotData, snapshotData.encryptedData);
|
|
2138
3325
|
}
|
|
2139
3326
|
if (!isValid) {
|
|
2140
3327
|
this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to replay without snapshot`);
|
|
@@ -2143,11 +3330,21 @@ class BaseMindooDB {
|
|
|
2143
3330
|
else {
|
|
2144
3331
|
const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
|
|
2145
3332
|
doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
3333
|
+
// Snapshot-head verification (docs/accesscontrol.md §10): decoded heads
|
|
3334
|
+
// must equal the declared snapshotHeadHashes or the snapshot is
|
|
3335
|
+
// discarded and the doc is replayed from individually-verified entries.
|
|
3336
|
+
if (!(0, materializationGuard_1.snapshotHeadsMatch)(automerge_adapter_1.Automerge.getHeads(doc), snapshotData.snapshotHeadHashes)) {
|
|
3337
|
+
this.logger.warn(`Snapshot ${snapshotData.id} heads do not match declared snapshotHeadHashes, falling back to replay without snapshot`);
|
|
3338
|
+
doc = undefined;
|
|
3339
|
+
startFromSnapshot = false;
|
|
3340
|
+
}
|
|
3341
|
+
else {
|
|
3342
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
|
|
3343
|
+
if (parsed) {
|
|
3344
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, snapshotData.id);
|
|
3345
|
+
}
|
|
3346
|
+
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
2149
3347
|
}
|
|
2150
|
-
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
2151
3348
|
}
|
|
2152
3349
|
}
|
|
2153
3350
|
}
|
|
@@ -2171,7 +3368,7 @@ class BaseMindooDB {
|
|
|
2171
3368
|
this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
|
|
2172
3369
|
continue;
|
|
2173
3370
|
}
|
|
2174
|
-
const isValid = await this.tenant.
|
|
3371
|
+
const isValid = await this.tenant.verifyEntrySignature(entryData, entryData.encryptedData);
|
|
2175
3372
|
if (!isValid) {
|
|
2176
3373
|
this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
|
|
2177
3374
|
continue;
|
|
@@ -2241,7 +3438,7 @@ class BaseMindooDB {
|
|
|
2241
3438
|
continue;
|
|
2242
3439
|
}
|
|
2243
3440
|
// Verify signature
|
|
2244
|
-
const isValid = await this.tenant.
|
|
3441
|
+
const isValid = await this.tenant.verifyEntrySignature(entryData, entryData.encryptedData);
|
|
2245
3442
|
if (!isValid) {
|
|
2246
3443
|
this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
|
|
2247
3444
|
continue;
|
|
@@ -2309,6 +3506,462 @@ class BaseMindooDB {
|
|
|
2309
3506
|
}
|
|
2310
3507
|
}
|
|
2311
3508
|
}
|
|
3509
|
+
async *iterateChangeRevisionsSince(cursor) {
|
|
3510
|
+
yield* this.iterateRevisionsInternal(null, cursor);
|
|
3511
|
+
}
|
|
3512
|
+
async *iterateDocRevisionsSince(docId, cursor) {
|
|
3513
|
+
yield* this.iterateRevisionsInternal(docId, cursor);
|
|
3514
|
+
}
|
|
3515
|
+
/**
|
|
3516
|
+
* Shared implementation of the revision-grain changefeed
|
|
3517
|
+
* ({@link iterateChangeRevisionsSince} / {@link iterateDocRevisionsSince}).
|
|
3518
|
+
*
|
|
3519
|
+
* Two distinct orderings drive the feed (docs/accesscontrol.md §8):
|
|
3520
|
+
*
|
|
3521
|
+
* - **Discovery (receiptOrder).** New entries are discovered via the store's
|
|
3522
|
+
* gap-free `scanEntriesSince(cursor)` scan in `(receiptOrder, id)` order, so
|
|
3523
|
+
* an incremental advance reads only entries appended since the cursor and
|
|
3524
|
+
* never re-scans cold documents. The resumable {@link RevisionCursor} is this
|
|
3525
|
+
* scan position, parked just before the earliest un-witnessed entry so the
|
|
3526
|
+
* provisional head overlay is re-evaluated on every resume.
|
|
3527
|
+
* - **Materialization (trusted time).** Each affected document's changes are
|
|
3528
|
+
* folded in trusted-time order, so every yielded revision carries the
|
|
3529
|
+
* document's *trusted-time-bounded merge* — the merge of all of the doc's
|
|
3530
|
+
* changes with `trustedTime <= this revision's trustedTime`. This is correct
|
|
3531
|
+
* at the head and at every intermediate node, and concurrent same-base
|
|
3532
|
+
* changes merge from the larger-trusted-time instant onward.
|
|
3533
|
+
*
|
|
3534
|
+
* Trusted-time rule: witnessed entries use their witness `receivedAt`;
|
|
3535
|
+
* un-witnessed local entries use the current wall clock, so they always sit at
|
|
3536
|
+
* the provisional head and never appear in past-time reads until witnessed.
|
|
3537
|
+
*
|
|
3538
|
+
* An in-memory per-doc LRU ({@link revisionFeedDocCache}) lets the common
|
|
3539
|
+
* in-order advance seed from cached state and decrypt only the new suffix; a
|
|
3540
|
+
* cache miss reconstructs the seed from the best snapshot whose coverage is
|
|
3541
|
+
* within the fold prefix via {@link computeDocumentMaterializationPlan}.
|
|
3542
|
+
*/
|
|
3543
|
+
async *iterateRevisionsInternal(docIdFilter, cursor) {
|
|
3544
|
+
// ── Phase 1: discover new entries in receiptOrder ─────────────────────
|
|
3545
|
+
const { newEntries, resumeCursor } = await this.discoverRevisionEntries(docIdFilter, cursor);
|
|
3546
|
+
if (newEntries.length === 0) {
|
|
3547
|
+
return;
|
|
3548
|
+
}
|
|
3549
|
+
// Trusted time for un-witnessed entries is the current wall clock, computed
|
|
3550
|
+
// once per run so all un-witnessed entries share a stable head instant.
|
|
3551
|
+
const now = Date.now();
|
|
3552
|
+
// Affected documents (those with at least one newly-discovered entry), and
|
|
3553
|
+
// the set of new entry ids per doc (the trusted-time positions from which the
|
|
3554
|
+
// doc's merged state may have changed and must be re-folded / re-emitted).
|
|
3555
|
+
const newIdsByDoc = new Map();
|
|
3556
|
+
for (const em of newEntries) {
|
|
3557
|
+
let set = newIdsByDoc.get(em.docId);
|
|
3558
|
+
if (!set) {
|
|
3559
|
+
set = new Set();
|
|
3560
|
+
newIdsByDoc.set(em.docId, set);
|
|
3561
|
+
}
|
|
3562
|
+
set.add(em.id);
|
|
3563
|
+
}
|
|
3564
|
+
// ── Phase 2: per affected doc, fold in trusted-time order, emit suffix ──
|
|
3565
|
+
for (const docId of Array.from(newIdsByDoc.keys()).sort()) {
|
|
3566
|
+
yield* this.foldDocRevisions(docId, newIdsByDoc.get(docId), now, resumeCursor);
|
|
3567
|
+
}
|
|
3568
|
+
}
|
|
3569
|
+
/**
|
|
3570
|
+
* Discovery phase of {@link iterateRevisionsInternal}: walk the store in
|
|
3571
|
+
* `(receiptOrder, id)` order from `cursor`, collecting the newly-appended
|
|
3572
|
+
* replay entries and computing the resume cursor.
|
|
3573
|
+
*
|
|
3574
|
+
* The resume cursor is the witnessed stable-prefix watermark: it advances over
|
|
3575
|
+
* leading witnessed entries but stops just before the first un-witnessed entry,
|
|
3576
|
+
* so un-witnessed entries (whose trusted time is provisional `now`) are
|
|
3577
|
+
* re-discovered and re-evaluated as a head overlay on the next resume.
|
|
3578
|
+
*/
|
|
3579
|
+
async discoverRevisionEntries(docIdFilter, cursor) {
|
|
3580
|
+
const replayTypes = [
|
|
3581
|
+
"doc_create",
|
|
3582
|
+
"doc_change",
|
|
3583
|
+
"doc_delete",
|
|
3584
|
+
"doc_undelete",
|
|
3585
|
+
];
|
|
3586
|
+
const newEntries = [];
|
|
3587
|
+
// Never regress below the input cursor; only advance while still in the
|
|
3588
|
+
// leading witnessed prefix.
|
|
3589
|
+
let stableCursor = cursor;
|
|
3590
|
+
let sawProvisional = false;
|
|
3591
|
+
const consider = (em) => {
|
|
3592
|
+
if (!this.isDocumentReplayEntry(em)) {
|
|
3593
|
+
return;
|
|
3594
|
+
}
|
|
3595
|
+
// Admin-only mode: only accept entries signed by the admin key (mirrors
|
|
3596
|
+
// materialization / iterateDocumentHistory). Non-admin entries are
|
|
3597
|
+
// permanently ignored and do not gate the stable cursor.
|
|
3598
|
+
if (this._isAdminOnlyDb && em.createdByPublicKey !== this.getAdminPublicKey()) {
|
|
3599
|
+
return;
|
|
3600
|
+
}
|
|
3601
|
+
newEntries.push(em);
|
|
3602
|
+
// The resume cursor may only advance through the leading prefix of
|
|
3603
|
+
// STABLE entries (whose trusted time never moves): witnessed entries and
|
|
3604
|
+
// legacy un-witnessed entries (stable at createdAt). It must park before
|
|
3605
|
+
// the first PROVISIONAL entry (versioned + un-witnessed, trusted time =
|
|
3606
|
+
// now), because that entry's position will change once it is witnessed
|
|
3607
|
+
// and re-delivered with a fresh receiptOrder. See {@link isProvisional}.
|
|
3608
|
+
const provisional = (0, storeEntryTime_1.isProvisional)(em);
|
|
3609
|
+
if (!sawProvisional) {
|
|
3610
|
+
if (!provisional && em.receiptOrder !== undefined) {
|
|
3611
|
+
stableCursor = { receiptOrder: em.receiptOrder, id: em.id };
|
|
3612
|
+
}
|
|
3613
|
+
else if (provisional) {
|
|
3614
|
+
sawProvisional = true;
|
|
3615
|
+
}
|
|
3616
|
+
}
|
|
3617
|
+
};
|
|
3618
|
+
if (this.supportsCursorScan(this.store)) {
|
|
3619
|
+
const baseFilters = { entryTypes: replayTypes };
|
|
3620
|
+
if (docIdFilter) {
|
|
3621
|
+
baseFilters.docId = docIdFilter;
|
|
3622
|
+
}
|
|
3623
|
+
const scanFilters = this.mergeTimeTravelScanFilters(baseFilters);
|
|
3624
|
+
let scanCursor = cursor
|
|
3625
|
+
? { receiptOrder: cursor.receiptOrder, id: cursor.id }
|
|
3626
|
+
: null;
|
|
3627
|
+
while (true) {
|
|
3628
|
+
const page = await this.store.scanEntriesSince(scanCursor, 1000, scanFilters);
|
|
3629
|
+
for (const em of page.entries) {
|
|
3630
|
+
consider(em);
|
|
3631
|
+
}
|
|
3632
|
+
scanCursor = page.nextCursor;
|
|
3633
|
+
if (!page.hasMore) {
|
|
3634
|
+
break;
|
|
3635
|
+
}
|
|
3636
|
+
}
|
|
3637
|
+
}
|
|
3638
|
+
else {
|
|
3639
|
+
// Fallback for stores without cursor scan: scan all metadata, order by
|
|
3640
|
+
// (receiptOrder, id), and take entries strictly after the cursor.
|
|
3641
|
+
const filters = docIdFilter
|
|
3642
|
+
? { docId: docIdFilter }
|
|
3643
|
+
: undefined;
|
|
3644
|
+
const all = (await this.scanAllMetadata(this.store, filters))
|
|
3645
|
+
.filter((em) => this.isDocumentReplayEntry(em))
|
|
3646
|
+
.sort((a, b) => this.compareReceiptOrder(a, b));
|
|
3647
|
+
for (const em of all) {
|
|
3648
|
+
if (cursor && !this.isAfterReceiptCursor(em, cursor)) {
|
|
3649
|
+
continue;
|
|
3650
|
+
}
|
|
3651
|
+
consider(em);
|
|
3652
|
+
}
|
|
3653
|
+
}
|
|
3654
|
+
return { newEntries, resumeCursor: stableCursor };
|
|
3655
|
+
}
|
|
3656
|
+
/** Compare two entries by `(receiptOrder, id)` ascending. */
|
|
3657
|
+
compareReceiptOrder(a, b) {
|
|
3658
|
+
const ra = a.receiptOrder ?? 0;
|
|
3659
|
+
const rb = b.receiptOrder ?? 0;
|
|
3660
|
+
return ra !== rb ? ra - rb : a.id.localeCompare(b.id);
|
|
3661
|
+
}
|
|
3662
|
+
/** True when `em` is strictly after `cursor` in `(receiptOrder, id)` order. */
|
|
3663
|
+
isAfterReceiptCursor(em, cursor) {
|
|
3664
|
+
const ro = em.receiptOrder ?? 0;
|
|
3665
|
+
if (ro !== cursor.receiptOrder) {
|
|
3666
|
+
return ro > cursor.receiptOrder;
|
|
3667
|
+
}
|
|
3668
|
+
return em.id.localeCompare(cursor.id) > 0;
|
|
3669
|
+
}
|
|
3670
|
+
/**
|
|
3671
|
+
* Materialization phase for a single affected document: fold its changes in
|
|
3672
|
+
* trusted-time order and yield one revision per change from the earliest
|
|
3673
|
+
* newly-discovered (or superseded) position onward, each carrying the
|
|
3674
|
+
* trusted-time-bounded merged document state.
|
|
3675
|
+
*/
|
|
3676
|
+
async *foldDocRevisions(docId, newIds, now, resumeCursor) {
|
|
3677
|
+
// All metadata for this doc (metadata only, cheap). Includes snapshots, used
|
|
3678
|
+
// for cache-miss seed reconstruction.
|
|
3679
|
+
const docAllMeta = await this.scanAllMetadata(this.store, { docId });
|
|
3680
|
+
let replayMeta = docAllMeta.filter((em) => this.isDocumentReplayEntry(em));
|
|
3681
|
+
if (this._isAdminOnlyDb) {
|
|
3682
|
+
const adminKey = this.getAdminPublicKey();
|
|
3683
|
+
replayMeta = replayMeta.filter((em) => em.createdByPublicKey === adminKey);
|
|
3684
|
+
}
|
|
3685
|
+
if (replayMeta.length === 0) {
|
|
3686
|
+
return;
|
|
3687
|
+
}
|
|
3688
|
+
// Order the doc's changes by trusted time (version-aware rule, see
|
|
3689
|
+
// {@link entryTrustedTime}): witnessed -> receivedAt; versioned un-witnessed
|
|
3690
|
+
// floats to the provisional head (now); legacy un-witnessed stays at its
|
|
3691
|
+
// stable createdAt.
|
|
3692
|
+
const ttSorted = replayMeta
|
|
3693
|
+
.map((meta) => ({
|
|
3694
|
+
meta,
|
|
3695
|
+
tt: (0, storeEntryTime_1.entryTrustedTime)(meta, now),
|
|
3696
|
+
witnessed: meta.receivedAt !== undefined,
|
|
3697
|
+
}))
|
|
3698
|
+
.sort((a, b) => (a.tt !== b.tt ? a.tt - b.tt : a.meta.id.localeCompare(b.meta.id)));
|
|
3699
|
+
// First trusted-time position touched by a newly-discovered entry. From here
|
|
3700
|
+
// the doc's merged state (and every later revision) may have changed, so we
|
|
3701
|
+
// re-fold and re-emit the suffix; earlier revisions are unchanged.
|
|
3702
|
+
const firstChangedIdx = ttSorted.findIndex((r) => newIds.has(r.meta.id));
|
|
3703
|
+
if (firstChangedIdx < 0) {
|
|
3704
|
+
return;
|
|
3705
|
+
}
|
|
3706
|
+
// Origin time and key id come from the document's create entry, independent
|
|
3707
|
+
// of fold order (under the trusted-time rule un-witnessed entries can sort
|
|
3708
|
+
// before the create, so we must not rely on seeing the create first).
|
|
3709
|
+
const createMeta = replayMeta.find((m) => m.entryType === "doc_create");
|
|
3710
|
+
// Seed the running doc to the merge of the unchanged prefix.
|
|
3711
|
+
const prefix = ttSorted.slice(0, firstChangedIdx).map((r) => r.meta);
|
|
3712
|
+
const seed = await this.seedRevisionFold(docId, prefix, docAllMeta);
|
|
3713
|
+
let doc = seed.doc ?? automerge_adapter_1.Automerge.init();
|
|
3714
|
+
const createdAt = createMeta?.createdAt ?? seed.createdAt ?? 0;
|
|
3715
|
+
const decryptionKeyId = createMeta?.decryptionKeyId ?? seed.decryptionKeyId ?? "default";
|
|
3716
|
+
// Cumulative metadata up to and including each revision (deletion state).
|
|
3717
|
+
const seenMeta = [...prefix];
|
|
3718
|
+
// Load + verify + decrypt the suffix in trusted-time-order windows: one
|
|
3719
|
+
// getEntries round trip per window instead of one per revision (a real win
|
|
3720
|
+
// for network/on-disk stores), and verify/decrypt the whole window in
|
|
3721
|
+
// parallel (independent per-entry crypto). Folding (applyChanges) below
|
|
3722
|
+
// still runs strictly sequentially in trusted-time order. The window also
|
|
3723
|
+
// bounds peak memory and response size (the suffix is the whole history on a
|
|
3724
|
+
// snapshot-less cold-start re-fold).
|
|
3725
|
+
let preparedById = new Map();
|
|
3726
|
+
let windowEnd = firstChangedIdx;
|
|
3727
|
+
for (let i = firstChangedIdx; i < ttSorted.length; i++) {
|
|
3728
|
+
const rev = ttSorted[i];
|
|
3729
|
+
const meta = rev.meta;
|
|
3730
|
+
if (i >= windowEnd) {
|
|
3731
|
+
const windowMetas = ttSorted.slice(i, i + this.revisionFoldLoadWindow);
|
|
3732
|
+
windowEnd = i + windowMetas.length;
|
|
3733
|
+
const batch = await this.store.getEntries(windowMetas.map((r) => r.meta.id));
|
|
3734
|
+
const prepared = await Promise.all(batch.map(async (entry) => {
|
|
3735
|
+
const isValid = await this.tenant.verifyEntrySignature(entry, entry.encryptedData);
|
|
3736
|
+
if (!isValid) {
|
|
3737
|
+
this.logger.warn(`Invalid signature for revision entry ${entry.id}, skipping`);
|
|
3738
|
+
return null;
|
|
3739
|
+
}
|
|
3740
|
+
const payload = await this.tenant.decryptPayload(entry.encryptedData, entry.decryptionKeyId);
|
|
3741
|
+
return { entry, payload };
|
|
3742
|
+
}));
|
|
3743
|
+
preparedById = new Map();
|
|
3744
|
+
for (const p of prepared) {
|
|
3745
|
+
if (p) {
|
|
3746
|
+
preparedById.set(p.entry.id, p);
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
3750
|
+
const ready = preparedById.get(meta.id);
|
|
3751
|
+
if (!ready) {
|
|
3752
|
+
// Absent from the loaded window means missing in the store or it failed
|
|
3753
|
+
// signature verification (already logged during the parallel prepare).
|
|
3754
|
+
this.logger.warn(`Revision entry ${meta.id} unavailable (missing or invalid), skipping`);
|
|
3755
|
+
continue;
|
|
3756
|
+
}
|
|
3757
|
+
const entryData = ready.entry;
|
|
3758
|
+
const decryptedPayload = ready.payload;
|
|
3759
|
+
// applyChanges buffers changes with unmet causal deps until they arrive,
|
|
3760
|
+
// so a change folded before its doc_create (possible under the trusted-time
|
|
3761
|
+
// rule for same-instant un-witnessed entries) still converges.
|
|
3762
|
+
const [appliedDoc] = automerge_adapter_1.Automerge.applyChanges(doc, [decryptedPayload]);
|
|
3763
|
+
doc = appliedDoc;
|
|
3764
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
|
|
3765
|
+
if (parsed) {
|
|
3766
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
|
|
3767
|
+
}
|
|
3768
|
+
seenMeta.push(meta);
|
|
3769
|
+
const internalDoc = {
|
|
3770
|
+
id: docId,
|
|
3771
|
+
doc: automerge_adapter_1.Automerge.clone(doc),
|
|
3772
|
+
createdAt,
|
|
3773
|
+
lastModified: meta.createdAt,
|
|
3774
|
+
decryptionKeyId,
|
|
3775
|
+
// Deletion as of this revision = latest reachable lifecycle entry among
|
|
3776
|
+
// this document's replay entries seen up to and including this one.
|
|
3777
|
+
isDeleted: this.computeIsDeletedFromMetadata([...seenMeta]),
|
|
3778
|
+
};
|
|
3779
|
+
yield {
|
|
3780
|
+
docId,
|
|
3781
|
+
entryId: meta.id,
|
|
3782
|
+
doc: this.wrapDocument(internalDoc),
|
|
3783
|
+
entryType: meta.entryType,
|
|
3784
|
+
trustedTime: rev.tt,
|
|
3785
|
+
witnessed: rev.witnessed,
|
|
3786
|
+
createdByPublicKey: meta.createdByPublicKey,
|
|
3787
|
+
cursor: resumeCursor,
|
|
3788
|
+
};
|
|
3789
|
+
}
|
|
3790
|
+
// Cache the doc's full merged state for the next incremental advance.
|
|
3791
|
+
this.putRevisionFeedCache(docId, {
|
|
3792
|
+
doc,
|
|
3793
|
+
coveredIds: new Set(ttSorted.map((r) => r.meta.id)),
|
|
3794
|
+
createdAt,
|
|
3795
|
+
decryptionKeyId,
|
|
3796
|
+
});
|
|
3797
|
+
}
|
|
3798
|
+
/**
|
|
3799
|
+
* Reconstruct the merged state of a document covering exactly the replay
|
|
3800
|
+
* entries in `prefix` (the unchanged head of a trusted-time fold).
|
|
3801
|
+
*
|
|
3802
|
+
* Fast path: if the LRU holds a merged doc whose coverage is a subset of the
|
|
3803
|
+
* prefix, clone it and apply only the missing entries. Otherwise reconstruct
|
|
3804
|
+
* from the best snapshot whose coverage is fully within the prefix (so the
|
|
3805
|
+
* snapshot can never pull in changes beyond the fold point) plus the uncovered
|
|
3806
|
+
* prefix tail, via {@link computeDocumentMaterializationPlan}.
|
|
3807
|
+
*/
|
|
3808
|
+
async seedRevisionFold(docId, prefix, docAllMeta) {
|
|
3809
|
+
if (prefix.length === 0) {
|
|
3810
|
+
return { doc: null, createdAt: null, decryptionKeyId: null };
|
|
3811
|
+
}
|
|
3812
|
+
const createEntry = prefix.find((m) => m.entryType === "doc_create");
|
|
3813
|
+
const createdAt = createEntry ? createEntry.createdAt : null;
|
|
3814
|
+
const decryptionKeyId = createEntry ? createEntry.decryptionKeyId : null;
|
|
3815
|
+
const prefixIds = new Set(prefix.map((m) => m.id));
|
|
3816
|
+
// LRU fast path: cached coverage is a subset of the prefix -> reuse + apply
|
|
3817
|
+
// only the missing entries (no snapshot/genesis reconstruction).
|
|
3818
|
+
const cached = this.getRevisionFeedCache(docId);
|
|
3819
|
+
if (cached && this.isIdSubset(cached.coveredIds, prefixIds)) {
|
|
3820
|
+
const missing = prefix.filter((m) => !cached.coveredIds.has(m.id));
|
|
3821
|
+
const doc = await this.applyChangeEntries(docId, automerge_adapter_1.Automerge.clone(cached.doc), missing);
|
|
3822
|
+
return {
|
|
3823
|
+
doc,
|
|
3824
|
+
createdAt: createdAt ?? cached.createdAt,
|
|
3825
|
+
decryptionKeyId: decryptionKeyId ?? cached.decryptionKeyId,
|
|
3826
|
+
};
|
|
3827
|
+
}
|
|
3828
|
+
// Cache miss: reconstruct from the best in-prefix snapshot + uncovered tail.
|
|
3829
|
+
const prefixSnapshots = docAllMeta.filter((m) => m.entryType === "doc_snapshot" && this.snapshotRootsWithin(m, prefixIds));
|
|
3830
|
+
const planInput = [...prefix, ...prefixSnapshots];
|
|
3831
|
+
const plan = (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, planInput);
|
|
3832
|
+
const metadataById = new Map(planInput.map((m) => [m.id, m]));
|
|
3833
|
+
let doc = null;
|
|
3834
|
+
if (plan.snapshotEntryId) {
|
|
3835
|
+
const snapMeta = metadataById.get(plan.snapshotEntryId);
|
|
3836
|
+
if (snapMeta) {
|
|
3837
|
+
doc = await this.loadSnapshotForFold(docId, snapMeta);
|
|
3838
|
+
}
|
|
3839
|
+
}
|
|
3840
|
+
if (!doc) {
|
|
3841
|
+
doc = automerge_adapter_1.Automerge.init();
|
|
3842
|
+
}
|
|
3843
|
+
const tail = plan.entryIdsToApply
|
|
3844
|
+
.map((id) => metadataById.get(id))
|
|
3845
|
+
.filter((m) => m !== undefined);
|
|
3846
|
+
doc = await this.applyChangeEntries(docId, doc, tail);
|
|
3847
|
+
return { doc, createdAt, decryptionKeyId };
|
|
3848
|
+
}
|
|
3849
|
+
/** True when every id in `sub` is contained in `sup`. */
|
|
3850
|
+
isIdSubset(sub, sup) {
|
|
3851
|
+
if (sub.size > sup.size) {
|
|
3852
|
+
return false;
|
|
3853
|
+
}
|
|
3854
|
+
for (const id of sub) {
|
|
3855
|
+
if (!sup.has(id)) {
|
|
3856
|
+
return false;
|
|
3857
|
+
}
|
|
3858
|
+
}
|
|
3859
|
+
return true;
|
|
3860
|
+
}
|
|
3861
|
+
/**
|
|
3862
|
+
* True when all of a snapshot's coverage roots are within `idSet`, i.e. the
|
|
3863
|
+
* snapshot represents a state reachable from entries inside the set (safe to
|
|
3864
|
+
* use as a seed without pulling in later changes).
|
|
3865
|
+
*/
|
|
3866
|
+
snapshotRootsWithin(snapshotMeta, idSet) {
|
|
3867
|
+
const roots = snapshotMeta.snapshotHeadEntryIds && snapshotMeta.snapshotHeadEntryIds.length > 0
|
|
3868
|
+
? snapshotMeta.snapshotHeadEntryIds
|
|
3869
|
+
: snapshotMeta.dependencyIds;
|
|
3870
|
+
if (!roots || roots.length === 0) {
|
|
3871
|
+
return false;
|
|
3872
|
+
}
|
|
3873
|
+
return roots.every((r) => idSet.has(r));
|
|
3874
|
+
}
|
|
3875
|
+
/** Load, verify, and decrypt a snapshot into an Automerge doc, or `null`. */
|
|
3876
|
+
async loadSnapshotForFold(docId, snapshotMeta) {
|
|
3877
|
+
const entries = await this.store.getEntries([snapshotMeta.id]);
|
|
3878
|
+
if (entries.length === 0) {
|
|
3879
|
+
return null;
|
|
3880
|
+
}
|
|
3881
|
+
const snapshotData = entries[0];
|
|
3882
|
+
if (this._isAdminOnlyDb && snapshotData.createdByPublicKey !== this.getAdminPublicKey()) {
|
|
3883
|
+
this.logger.warn(`Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
|
|
3884
|
+
return null;
|
|
3885
|
+
}
|
|
3886
|
+
const isValid = await this.tenant.verifyEntrySignature(snapshotData, snapshotData.encryptedData);
|
|
3887
|
+
if (!isValid) {
|
|
3888
|
+
this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to replay`);
|
|
3889
|
+
return null;
|
|
3890
|
+
}
|
|
3891
|
+
const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
|
|
3892
|
+
const doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
|
|
3893
|
+
// Mandatory snapshot-head verification (docs/accesscontrol.md §10): the
|
|
3894
|
+
// decoded snapshot's Automerge heads must exactly equal the covered heads
|
|
3895
|
+
// declared in the signed metadata, so a snapshot cannot smuggle content the
|
|
3896
|
+
// author was not allowed to write. On mismatch, fall back to replay.
|
|
3897
|
+
if (!(0, materializationGuard_1.snapshotHeadsMatch)(automerge_adapter_1.Automerge.getHeads(doc), snapshotData.snapshotHeadHashes)) {
|
|
3898
|
+
this.logger.warn(`Snapshot ${snapshotData.id} heads do not match declared snapshotHeadHashes, falling back to replay`);
|
|
3899
|
+
return null;
|
|
3900
|
+
}
|
|
3901
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
|
|
3902
|
+
if (parsed) {
|
|
3903
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, snapshotData.id);
|
|
3904
|
+
}
|
|
3905
|
+
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
3906
|
+
return doc;
|
|
3907
|
+
}
|
|
3908
|
+
/**
|
|
3909
|
+
* Load, verify, decrypt, and apply a set of replay change entries onto `doc`.
|
|
3910
|
+
* `Automerge.applyChanges` buffers changes whose causal dependencies are not
|
|
3911
|
+
* yet present, so the entries need not be supplied in dependency order.
|
|
3912
|
+
*/
|
|
3913
|
+
async applyChangeEntries(docId, doc, metas) {
|
|
3914
|
+
if (metas.length === 0) {
|
|
3915
|
+
return doc;
|
|
3916
|
+
}
|
|
3917
|
+
const loaded = await this.store.getEntries(metas.map((m) => m.id));
|
|
3918
|
+
const byId = new Map(loaded.map((e) => [e.id, e]));
|
|
3919
|
+
const changes = [];
|
|
3920
|
+
for (const meta of metas) {
|
|
3921
|
+
const entryData = byId.get(meta.id);
|
|
3922
|
+
if (!entryData) {
|
|
3923
|
+
this.logger.warn(`Revision entry ${meta.id} not found in store, skipping`);
|
|
3924
|
+
continue;
|
|
3925
|
+
}
|
|
3926
|
+
const isValid = await this.tenant.verifyEntrySignature(entryData, entryData.encryptedData);
|
|
3927
|
+
if (!isValid) {
|
|
3928
|
+
this.logger.warn(`Invalid signature for revision entry ${entryData.id}, skipping`);
|
|
3929
|
+
continue;
|
|
3930
|
+
}
|
|
3931
|
+
const decrypted = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
|
|
3932
|
+
changes.push(decrypted);
|
|
3933
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
|
|
3934
|
+
if (parsed) {
|
|
3935
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
|
|
3936
|
+
}
|
|
3937
|
+
}
|
|
3938
|
+
if (changes.length === 0) {
|
|
3939
|
+
return doc;
|
|
3940
|
+
}
|
|
3941
|
+
const [applied] = automerge_adapter_1.Automerge.applyChanges(doc, changes);
|
|
3942
|
+
return applied;
|
|
3943
|
+
}
|
|
3944
|
+
/** Touch the revision-feed LRU, moving `docId` to most-recently-used. */
|
|
3945
|
+
getRevisionFeedCache(docId) {
|
|
3946
|
+
const state = this.revisionFeedDocCache.get(docId);
|
|
3947
|
+
if (state) {
|
|
3948
|
+
this.revisionFeedDocCache.delete(docId);
|
|
3949
|
+
this.revisionFeedDocCache.set(docId, state);
|
|
3950
|
+
}
|
|
3951
|
+
return state;
|
|
3952
|
+
}
|
|
3953
|
+
/** Insert/update the revision-feed LRU, evicting the least-recently-used. */
|
|
3954
|
+
putRevisionFeedCache(docId, state) {
|
|
3955
|
+
this.revisionFeedDocCache.delete(docId);
|
|
3956
|
+
this.revisionFeedDocCache.set(docId, state);
|
|
3957
|
+
while (this.revisionFeedDocCache.size > this.revisionFeedCacheLimit) {
|
|
3958
|
+
const oldest = this.revisionFeedDocCache.keys().next().value;
|
|
3959
|
+
if (oldest === undefined) {
|
|
3960
|
+
break;
|
|
3961
|
+
}
|
|
3962
|
+
this.revisionFeedDocCache.delete(oldest);
|
|
3963
|
+
}
|
|
3964
|
+
}
|
|
2312
3965
|
async getDocumentHistoryPage(docId, options) {
|
|
2313
3966
|
const startedAt = Date.now();
|
|
2314
3967
|
const limit = Math.max(1, Math.floor(options?.limit ?? 100));
|
|
@@ -2395,7 +4048,7 @@ class BaseMindooDB {
|
|
|
2395
4048
|
result.set(metadata.id, null);
|
|
2396
4049
|
continue;
|
|
2397
4050
|
}
|
|
2398
|
-
const isValid = await this.tenant.
|
|
4051
|
+
const isValid = await this.tenant.verifyEntrySignature(entry, entry.encryptedData);
|
|
2399
4052
|
if (!isValid) {
|
|
2400
4053
|
result.set(metadata.id, null);
|
|
2401
4054
|
continue;
|
|
@@ -2518,7 +4171,7 @@ class BaseMindooDB {
|
|
|
2518
4171
|
this.logger.warn(`Admin-only DB: skipping DAG details for ${entry.id} not signed by admin key`);
|
|
2519
4172
|
}
|
|
2520
4173
|
else {
|
|
2521
|
-
isValid = await this.tenant.
|
|
4174
|
+
isValid = await this.tenant.verifyEntrySignature(entry, entry.encryptedData);
|
|
2522
4175
|
}
|
|
2523
4176
|
if (isValid) {
|
|
2524
4177
|
const decryptedPayload = await this.tenant.decryptPayload(entry.encryptedData, entry.decryptionKeyId);
|
|
@@ -3215,7 +4868,7 @@ class BaseMindooDB {
|
|
|
3215
4868
|
this.logger.warn(`Admin-only DB: skipping conflict analysis entry ${entry.id} not signed by admin key`);
|
|
3216
4869
|
continue;
|
|
3217
4870
|
}
|
|
3218
|
-
const isValid = await this.tenant.
|
|
4871
|
+
const isValid = await this.tenant.verifyEntrySignature(entry, entry.encryptedData);
|
|
3219
4872
|
if (!isValid) {
|
|
3220
4873
|
this.logger.warn(`Invalid signature for conflict analysis entry ${entry.id}, skipping`);
|
|
3221
4874
|
continue;
|
|
@@ -3668,13 +5321,73 @@ class BaseMindooDB {
|
|
|
3668
5321
|
return docIds;
|
|
3669
5322
|
}
|
|
3670
5323
|
async deleteDocument(docId, options = {}) {
|
|
3671
|
-
return this.deleteDocInternal(docId, options.signingKeyPair, options.signingKeyPassword);
|
|
5324
|
+
return this.deleteDocInternal(docId, options.signingKeyPair, options.signingKeyPassword, options.bypassAccessControlPrecheck);
|
|
5325
|
+
}
|
|
5326
|
+
/**
|
|
5327
|
+
* Bulk-delete documents in a single append-only store write.
|
|
5328
|
+
*
|
|
5329
|
+
* Semantically equivalent to calling {@link deleteDocument} once per id, but
|
|
5330
|
+
* all `doc_delete` lifecycle entries are written through ONE `putEntries()`
|
|
5331
|
+
* call (a single readwrite transaction on IndexedDB-backed stores), and the
|
|
5332
|
+
* deleted documents are dropped from the L1 cache instead of being
|
|
5333
|
+
* dirty-tracked (the entries are already durable, so no L2 flush is needed).
|
|
5334
|
+
*
|
|
5335
|
+
* Documents that are missing or already deleted are skipped (idempotent
|
|
5336
|
+
* bulk semantics: a re-run of a destructive import must not fail because a
|
|
5337
|
+
* previous run already removed some documents). Any other failure, including
|
|
5338
|
+
* a denied ACL precheck, rejects the whole call before anything is written.
|
|
5339
|
+
*/
|
|
5340
|
+
async deleteDocuments(docIds, options = {}) {
|
|
5341
|
+
this.assertWritable("deleteDocuments");
|
|
5342
|
+
if (docIds.length === 0) {
|
|
5343
|
+
return;
|
|
5344
|
+
}
|
|
5345
|
+
await this.assertLifecycleMutationAllowed(options.signingKeyPair, options.signingKeyPassword);
|
|
5346
|
+
this.logger.info(`Bulk-deleting ${docIds.length} documents`);
|
|
5347
|
+
// Unlike creation, a delete must first materialize the CURRENT document
|
|
5348
|
+
// (the doc_delete entry is an Automerge change on its heads), which costs
|
|
5349
|
+
// an IndexedDB metadata scan plus per-entry decrypt + signature verify.
|
|
5350
|
+
// Those are independent per document and dominated by IndexedDB/WebCrypto
|
|
5351
|
+
// latency (both run off the JS thread), so prepare with bounded
|
|
5352
|
+
// concurrency instead of strictly sequentially. Results keep input order;
|
|
5353
|
+
// any failure rejects the whole batch before anything is written
|
|
5354
|
+
// (fail-fast, same as the sequential version).
|
|
5355
|
+
const preparedByIndex = new Array(docIds.length).fill(null);
|
|
5356
|
+
const concurrency = Math.min(8, docIds.length);
|
|
5357
|
+
let nextIndex = 0;
|
|
5358
|
+
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
5359
|
+
while (true) {
|
|
5360
|
+
const index = nextIndex++;
|
|
5361
|
+
if (index >= docIds.length) {
|
|
5362
|
+
return;
|
|
5363
|
+
}
|
|
5364
|
+
const docId = docIds[index];
|
|
5365
|
+
const internalDoc = await this.loadDocumentInternal(docId);
|
|
5366
|
+
if (!internalDoc || internalDoc.isDeleted) {
|
|
5367
|
+
this.logger.debug(`Skipping bulk delete for ${docId} — not found or already deleted`);
|
|
5368
|
+
continue;
|
|
5369
|
+
}
|
|
5370
|
+
preparedByIndex[index] = await this.prepareLifecycleEntry(internalDoc, "doc_delete", options.signingKeyPair, options.signingKeyPassword, options.bypassAccessControlPrecheck);
|
|
5371
|
+
}
|
|
5372
|
+
}));
|
|
5373
|
+
const prepared = preparedByIndex.filter((p) => p !== null);
|
|
5374
|
+
if (prepared.length === 0) {
|
|
5375
|
+
return;
|
|
5376
|
+
}
|
|
5377
|
+
await this.store.putEntries(prepared.map((p) => p.fullEntry));
|
|
5378
|
+
for (const p of prepared) {
|
|
5379
|
+
await this.applyLifecycleEntryLocally(p, { markDirty: false });
|
|
5380
|
+
}
|
|
5381
|
+
// One metadata-checkpoint invalidation for the whole batch.
|
|
5382
|
+
this.cacheMetaDirty = true;
|
|
5383
|
+
this.cacheManager?.markDirty();
|
|
5384
|
+
this.logger.info(`Bulk-deleted ${prepared.length} documents (${docIds.length - prepared.length} skipped)`);
|
|
3672
5385
|
}
|
|
3673
5386
|
async deleteDocumentWithSigningKey(docId, signingKeyPair, signingKeyPassword) {
|
|
3674
5387
|
return this.deleteDocument(docId, { signingKeyPair, signingKeyPassword });
|
|
3675
5388
|
}
|
|
3676
5389
|
async undeleteDocument(docId, options = {}) {
|
|
3677
|
-
return this.undeleteDocInternal(docId, options.signingKeyPair, options.signingKeyPassword);
|
|
5390
|
+
return this.undeleteDocInternal(docId, options.signingKeyPair, options.signingKeyPassword, options.bypassAccessControlPrecheck);
|
|
3678
5391
|
}
|
|
3679
5392
|
async assertLifecycleMutationAllowed(signingKeyPair, signingKeyPassword) {
|
|
3680
5393
|
this.assertWritable("document lifecycle mutation");
|
|
@@ -3699,7 +5412,18 @@ class BaseMindooDB {
|
|
|
3699
5412
|
* change that advances `_lastModified`, wraps it in the requested lifecycle
|
|
3700
5413
|
* StoreEntry type, signs/encrypts it, and updates local caches/indexes.
|
|
3701
5414
|
*/
|
|
3702
|
-
async writeLifecycleEntry(internalDoc, entryType, signingKeyPair, signingKeyPassword) {
|
|
5415
|
+
async writeLifecycleEntry(internalDoc, entryType, signingKeyPair, signingKeyPassword, bypassPrecheck) {
|
|
5416
|
+
const prepared = await this.prepareLifecycleEntry(internalDoc, entryType, signingKeyPair, signingKeyPassword, bypassPrecheck);
|
|
5417
|
+
await this.store.putEntries([prepared.fullEntry]);
|
|
5418
|
+
await this.applyLifecycleEntryLocally(prepared, { markDirty: true });
|
|
5419
|
+
}
|
|
5420
|
+
/**
|
|
5421
|
+
* Build (but do not persist) a delete/undelete lifecycle entry for an
|
|
5422
|
+
* existing document, including the ACL precheck. Callers persist the
|
|
5423
|
+
* returned `fullEntry` (single or batched `putEntries`) and then call
|
|
5424
|
+
* {@link applyLifecycleEntryLocally} to update caches and indexes.
|
|
5425
|
+
*/
|
|
5426
|
+
async prepareLifecycleEntry(internalDoc, entryType, signingKeyPair, signingKeyPassword, bypassPrecheck) {
|
|
3703
5427
|
// Delete and undelete are encoded as normal Automerge changes so the DAG
|
|
3704
5428
|
// keeps causal ancestry, while the StoreEntry type carries lifecycle intent.
|
|
3705
5429
|
const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
|
|
@@ -3738,6 +5462,10 @@ class BaseMindooDB {
|
|
|
3738
5462
|
signature = await this.tenant.signPayload(encryptedPayload);
|
|
3739
5463
|
createdByPublicKey = currentUser.userSigningPublicKey;
|
|
3740
5464
|
}
|
|
5465
|
+
// Attachment snapshot for lifecycle entries: a delete marks the document
|
|
5466
|
+
// inactive, so its attachments are no longer referenced (empty set -> their
|
|
5467
|
+
// chunks become reclaim candidates); an undelete restores the current set.
|
|
5468
|
+
const lifecycleAttachmentRefs = entryType === "doc_delete" ? [] : this.collectAttachmentRefs(newDoc);
|
|
3741
5469
|
// The encrypted Automerge change is the payload; the metadata classifies it
|
|
3742
5470
|
// as a delete or undelete lifecycle entry for fast history/view queries.
|
|
3743
5471
|
const entryMetadata = {
|
|
@@ -3752,28 +5480,85 @@ class BaseMindooDB {
|
|
|
3752
5480
|
signature,
|
|
3753
5481
|
originalSize: changeBytes.length,
|
|
3754
5482
|
encryptedSize: encryptedPayload.length,
|
|
5483
|
+
attachmentRefs: lifecycleAttachmentRefs.length > 0 ? lifecycleAttachmentRefs : undefined,
|
|
5484
|
+
entryVersion: types_1.CURRENT_STORE_ENTRY_VERSION,
|
|
3755
5485
|
};
|
|
5486
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
5487
|
+
entryMetadata.metadataSignature = await this.computeEntryMetadataSignature(entryMetadata, useCustomSigningKey ? { signingKeyPair, signingKeyPassword } : undefined);
|
|
3756
5488
|
const fullEntry = {
|
|
3757
5489
|
...entryMetadata,
|
|
3758
5490
|
encryptedData: encryptedPayload,
|
|
3759
5491
|
};
|
|
3760
|
-
|
|
5492
|
+
// Client-side write-policy precheck (docs/accesscontrol.md §9). `internalDoc.doc`
|
|
5493
|
+
// is still the pre-change state here (reassigned in
|
|
5494
|
+
// applyLifecycleEntryLocally, after the store write). For deletes the rules
|
|
5495
|
+
// default to the "before" state; both states are supplied so either `when`
|
|
5496
|
+
// resolves. Throws AccessDeniedError if denied.
|
|
5497
|
+
{
|
|
5498
|
+
const creatorKey = await this.resolveCreatorSigningKey(docId);
|
|
5499
|
+
const beforeState = internalDoc.doc;
|
|
5500
|
+
await this.assertWriteAllowed({
|
|
5501
|
+
op: entryType,
|
|
5502
|
+
signerKey: createdByPublicKey,
|
|
5503
|
+
isAuthor: creatorKey !== null && createdByPublicKey === creatorKey,
|
|
5504
|
+
bypass: bypassPrecheck,
|
|
5505
|
+
getBeforeDoc: () => this.convertAutomergeToJS(beforeState),
|
|
5506
|
+
getAfterDoc: () => this.convertAutomergeToJS(newDoc),
|
|
5507
|
+
});
|
|
5508
|
+
}
|
|
5509
|
+
return {
|
|
5510
|
+
internalDoc,
|
|
5511
|
+
entryType,
|
|
5512
|
+
newDoc,
|
|
5513
|
+
automergeHash,
|
|
5514
|
+
entryId,
|
|
5515
|
+
entryMetadata,
|
|
5516
|
+
fullEntry,
|
|
5517
|
+
now,
|
|
5518
|
+
};
|
|
5519
|
+
}
|
|
5520
|
+
/**
|
|
5521
|
+
* Apply a persisted lifecycle entry (see {@link prepareLifecycleEntry}) to
|
|
5522
|
+
* the in-memory document, cache, and changefeed index. Must be called after
|
|
5523
|
+
* the entry was written to the store.
|
|
5524
|
+
*
|
|
5525
|
+
* With `markDirty: false` (bulk path) the document is dropped from the L1
|
|
5526
|
+
* cache instead of being dirty-tracked: the lifecycle entry is already
|
|
5527
|
+
* durable in the append-only store, so skipping the L2 flush avoids the
|
|
5528
|
+
* per-document flush-before-evict write storm during large batches.
|
|
5529
|
+
*/
|
|
5530
|
+
async applyLifecycleEntryLocally(prepared, options) {
|
|
5531
|
+
const { internalDoc, entryType, newDoc, automergeHash, entryId, entryMetadata, now } = prepared;
|
|
5532
|
+
const docId = internalDoc.id;
|
|
3761
5533
|
this.registerAutomergeHashMapping(docId, automergeHash, entryId);
|
|
3762
5534
|
// Keep the loaded document, cache, index, and sync dirty tracking in step
|
|
3763
5535
|
// with the append-only store entry we just wrote.
|
|
3764
5536
|
internalDoc.doc = newDoc;
|
|
3765
5537
|
internalDoc.isDeleted = entryType === "doc_delete";
|
|
3766
5538
|
internalDoc.lastModified = now;
|
|
3767
|
-
|
|
3768
|
-
|
|
3769
|
-
|
|
5539
|
+
// The delete/undelete entry just written is versioned and un-witnessed, so
|
|
5540
|
+
// the document is awaiting witness until pushed + stamped with a receipt.
|
|
5541
|
+
internalDoc.awaitingWitness = (0, storeEntryTime_1.isProvisional)(entryMetadata) || (internalDoc.awaitingWitness ?? false);
|
|
5542
|
+
// The doc is now witness-era (we just wrote a versioned entry), so it is
|
|
5543
|
+
// witnessed iff it is no longer awaiting a receipt.
|
|
5544
|
+
internalDoc.witnessed = (0, storeEntryTime_1.isVersioned)(entryMetadata) && !internalDoc.awaitingWitness;
|
|
5545
|
+
if (options.markDirty) {
|
|
5546
|
+
await this.storeCachedDocument(internalDoc);
|
|
5547
|
+
this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed);
|
|
5548
|
+
this.markDocDirty(docId);
|
|
5549
|
+
}
|
|
5550
|
+
else {
|
|
5551
|
+
this.docCache.delete(docId);
|
|
5552
|
+
this.dirtyDocIds.delete(docId);
|
|
5553
|
+
this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed);
|
|
5554
|
+
}
|
|
3770
5555
|
}
|
|
3771
5556
|
/**
|
|
3772
5557
|
* Internal method to delete a document.
|
|
3773
5558
|
* Handles both regular deletion (signed by current user) and
|
|
3774
5559
|
* deletion with a custom signing key (e.g., for directory operations).
|
|
3775
5560
|
*/
|
|
3776
|
-
async deleteDocInternal(docId, signingKeyPair, signingKeyPassword) {
|
|
5561
|
+
async deleteDocInternal(docId, signingKeyPair, signingKeyPassword, bypassPrecheck) {
|
|
3777
5562
|
const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
|
|
3778
5563
|
this.logger.debug(`Deleting document ${docId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
|
|
3779
5564
|
await this.assertLifecycleMutationAllowed(signingKeyPair, signingKeyPassword);
|
|
@@ -3782,26 +5567,26 @@ class BaseMindooDB {
|
|
|
3782
5567
|
if (!internalDoc || internalDoc.isDeleted) {
|
|
3783
5568
|
throw new Error(`Document ${docId} not found or already deleted`);
|
|
3784
5569
|
}
|
|
3785
|
-
await this.writeLifecycleEntry(internalDoc, "doc_delete", signingKeyPair, signingKeyPassword);
|
|
5570
|
+
await this.writeLifecycleEntry(internalDoc, "doc_delete", signingKeyPair, signingKeyPassword, bypassPrecheck);
|
|
3786
5571
|
this.logger.info(`Document ${docId} deleted successfully`);
|
|
3787
5572
|
}
|
|
3788
|
-
async undeleteDocInternal(docId, signingKeyPair, signingKeyPassword) {
|
|
5573
|
+
async undeleteDocInternal(docId, signingKeyPair, signingKeyPassword, bypassPrecheck) {
|
|
3789
5574
|
const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
|
|
3790
5575
|
this.logger.debug(`Undeleting document ${docId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
|
|
3791
5576
|
await this.assertLifecycleMutationAllowed(signingKeyPair, signingKeyPassword);
|
|
3792
5577
|
const internalDoc = await this.loadDocumentInternal(docId);
|
|
3793
5578
|
if (!internalDoc) {
|
|
3794
|
-
throw new
|
|
5579
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
3795
5580
|
}
|
|
3796
5581
|
if (!internalDoc.isDeleted) {
|
|
3797
5582
|
this.logger.debug(`Document ${docId} is already alive; undelete is a no-op`);
|
|
3798
5583
|
return;
|
|
3799
5584
|
}
|
|
3800
|
-
await this.writeLifecycleEntry(internalDoc, "doc_undelete", signingKeyPair, signingKeyPassword);
|
|
5585
|
+
await this.writeLifecycleEntry(internalDoc, "doc_undelete", signingKeyPair, signingKeyPassword, bypassPrecheck);
|
|
3801
5586
|
this.logger.info(`Document ${docId} undeleted successfully`);
|
|
3802
5587
|
}
|
|
3803
5588
|
async changeDoc(doc, changeFunc, options = {}) {
|
|
3804
|
-
return this.changeDocInternal(doc, changeFunc, options.signingKeyPair, options.signingKeyPassword);
|
|
5589
|
+
return this.changeDocInternal(doc, changeFunc, options.signingKeyPair, options.signingKeyPassword, options.bypassAccessControlPrecheck);
|
|
3805
5590
|
}
|
|
3806
5591
|
async changeDocWithSigningKey(doc, changeFunc, signingKeyPair, signingKeyPassword) {
|
|
3807
5592
|
return this.changeDoc(doc, changeFunc, { signingKeyPair, signingKeyPassword });
|
|
@@ -3821,12 +5606,12 @@ class BaseMindooDB {
|
|
|
3821
5606
|
if (!internalDoc) {
|
|
3822
5607
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
3823
5608
|
if (!loadedDoc) {
|
|
3824
|
-
throw new
|
|
5609
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
3825
5610
|
}
|
|
3826
5611
|
internalDoc = loadedDoc;
|
|
3827
5612
|
}
|
|
3828
5613
|
if (internalDoc.isDeleted) {
|
|
3829
|
-
throw new
|
|
5614
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
3830
5615
|
}
|
|
3831
5616
|
this.validateTextPatch(patch);
|
|
3832
5617
|
const now = Date.now();
|
|
@@ -3848,7 +5633,8 @@ class BaseMindooDB {
|
|
|
3848
5633
|
}
|
|
3849
5634
|
return automerge_adapter_1.Automerge.change(doc, applyEdits);
|
|
3850
5635
|
});
|
|
3851
|
-
|
|
5636
|
+
if (this.isDebugEnabled())
|
|
5637
|
+
this.logger.debug(`Successfully applied text patch, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
3852
5638
|
}
|
|
3853
5639
|
catch (error) {
|
|
3854
5640
|
this.logger.error(`Error applying text patch for document ${docId}:`, error);
|
|
@@ -3884,12 +5670,12 @@ class BaseMindooDB {
|
|
|
3884
5670
|
if (!internalDoc) {
|
|
3885
5671
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
3886
5672
|
if (!loadedDoc) {
|
|
3887
|
-
throw new
|
|
5673
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
3888
5674
|
}
|
|
3889
5675
|
internalDoc = loadedDoc;
|
|
3890
5676
|
}
|
|
3891
5677
|
if (internalDoc.isDeleted) {
|
|
3892
|
-
throw new
|
|
5678
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
3893
5679
|
}
|
|
3894
5680
|
this.validateRichTextPatch(patch);
|
|
3895
5681
|
const now = Date.now();
|
|
@@ -3922,7 +5708,8 @@ class BaseMindooDB {
|
|
|
3922
5708
|
}
|
|
3923
5709
|
return automerge_adapter_1.Automerge.change(doc, applySpans);
|
|
3924
5710
|
});
|
|
3925
|
-
|
|
5711
|
+
if (this.isDebugEnabled())
|
|
5712
|
+
this.logger.debug(`Successfully applied rich-text patch, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
3926
5713
|
}
|
|
3927
5714
|
catch (error) {
|
|
3928
5715
|
this.logger.error(`Error applying rich-text patch for document ${docId}:`, error);
|
|
@@ -3958,12 +5745,12 @@ class BaseMindooDB {
|
|
|
3958
5745
|
if (!internalDoc) {
|
|
3959
5746
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
3960
5747
|
if (!loadedDoc) {
|
|
3961
|
-
throw new
|
|
5748
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
3962
5749
|
}
|
|
3963
5750
|
internalDoc = loadedDoc;
|
|
3964
5751
|
}
|
|
3965
5752
|
if (internalDoc.isDeleted) {
|
|
3966
|
-
throw new
|
|
5753
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
3967
5754
|
}
|
|
3968
5755
|
this.validateRichTextStepPatch(patch);
|
|
3969
5756
|
const now = Date.now();
|
|
@@ -4008,7 +5795,8 @@ class BaseMindooDB {
|
|
|
4008
5795
|
}
|
|
4009
5796
|
return automerge_adapter_1.Automerge.change(doc, applySteps);
|
|
4010
5797
|
});
|
|
4011
|
-
|
|
5798
|
+
if (this.isDebugEnabled())
|
|
5799
|
+
this.logger.debug(`Successfully applied rich-text steps, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
4012
5800
|
}
|
|
4013
5801
|
catch (error) {
|
|
4014
5802
|
this.logger.error(`Error applying rich-text steps for document ${docId}:`, error);
|
|
@@ -4039,12 +5827,12 @@ class BaseMindooDB {
|
|
|
4039
5827
|
if (!internalDoc) {
|
|
4040
5828
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
4041
5829
|
if (!loadedDoc) {
|
|
4042
|
-
throw new
|
|
5830
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
4043
5831
|
}
|
|
4044
5832
|
internalDoc = loadedDoc;
|
|
4045
5833
|
}
|
|
4046
5834
|
if (internalDoc.isDeleted) {
|
|
4047
|
-
throw new
|
|
5835
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
4048
5836
|
}
|
|
4049
5837
|
const spans = automerge_adapter_1.Automerge.spans(internalDoc.doc, path);
|
|
4050
5838
|
return {
|
|
@@ -4079,12 +5867,12 @@ class BaseMindooDB {
|
|
|
4079
5867
|
if (!internalDoc) {
|
|
4080
5868
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
4081
5869
|
if (!loadedDoc) {
|
|
4082
|
-
throw new
|
|
5870
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
4083
5871
|
}
|
|
4084
5872
|
internalDoc = loadedDoc;
|
|
4085
5873
|
}
|
|
4086
5874
|
if (internalDoc.isDeleted) {
|
|
4087
|
-
throw new
|
|
5875
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
4088
5876
|
}
|
|
4089
5877
|
await this.hydrateAutomergeHashMappingsFromStore(docId);
|
|
4090
5878
|
const now = Date.now();
|
|
@@ -4102,7 +5890,9 @@ class BaseMindooDB {
|
|
|
4102
5890
|
const [mergedDoc] = automerge_adapter_1.Automerge.applyChanges(currentDoc, patch.changes.map((change) => new Uint8Array(change)));
|
|
4103
5891
|
return mergedDoc;
|
|
4104
5892
|
});
|
|
4105
|
-
|
|
5893
|
+
if (this.isDebugEnabled()) {
|
|
5894
|
+
this.logger.debug(`Successfully applied Automerge changes, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
5895
|
+
}
|
|
4106
5896
|
}
|
|
4107
5897
|
catch (error) {
|
|
4108
5898
|
this.logger.error(`Error applying Automerge changes for document ${docId}:`, error);
|
|
@@ -4160,12 +5950,12 @@ class BaseMindooDB {
|
|
|
4160
5950
|
if (!internalDoc) {
|
|
4161
5951
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
4162
5952
|
if (!loadedDoc) {
|
|
4163
|
-
throw new
|
|
5953
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
4164
5954
|
}
|
|
4165
5955
|
internalDoc = loadedDoc;
|
|
4166
5956
|
}
|
|
4167
5957
|
if (internalDoc.isDeleted) {
|
|
4168
|
-
throw new
|
|
5958
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
4169
5959
|
}
|
|
4170
5960
|
return internalDoc;
|
|
4171
5961
|
}
|
|
@@ -4184,12 +5974,12 @@ class BaseMindooDB {
|
|
|
4184
5974
|
if (!internalDoc) {
|
|
4185
5975
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
4186
5976
|
if (!loadedDoc) {
|
|
4187
|
-
throw new
|
|
5977
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
4188
5978
|
}
|
|
4189
5979
|
internalDoc = loadedDoc;
|
|
4190
5980
|
}
|
|
4191
5981
|
if (internalDoc.isDeleted) {
|
|
4192
|
-
throw new
|
|
5982
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
4193
5983
|
}
|
|
4194
5984
|
this.validateJsonPatch(patch);
|
|
4195
5985
|
const now = Date.now();
|
|
@@ -4207,7 +5997,8 @@ class BaseMindooDB {
|
|
|
4207
5997
|
}
|
|
4208
5998
|
return automerge_adapter_1.Automerge.change(doc, applyPatch);
|
|
4209
5999
|
});
|
|
4210
|
-
|
|
6000
|
+
if (this.isDebugEnabled())
|
|
6001
|
+
this.logger.debug(`Successfully applied JSON patch, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
4211
6002
|
}
|
|
4212
6003
|
catch (error) {
|
|
4213
6004
|
this.logger.error(`Error applying JSON patch for document ${docId}:`, error);
|
|
@@ -4228,7 +6019,7 @@ class BaseMindooDB {
|
|
|
4228
6019
|
data: wrapped.getData(),
|
|
4229
6020
|
};
|
|
4230
6021
|
}
|
|
4231
|
-
async changeDocInternal(doc, changeFunc, signingKeyPair, signingKeyPassword) {
|
|
6022
|
+
async changeDocInternal(doc, changeFunc, signingKeyPair, signingKeyPassword, bypassPrecheck) {
|
|
4232
6023
|
this.assertWritable("changeDoc");
|
|
4233
6024
|
const docId = doc.getId();
|
|
4234
6025
|
if ((signingKeyPair !== undefined) !== (signingKeyPassword !== undefined)) {
|
|
@@ -4252,7 +6043,7 @@ class BaseMindooDB {
|
|
|
4252
6043
|
this.logger.debug(`Document ${docId} not in cache, loading from store`);
|
|
4253
6044
|
const loadedDoc = await this.loadDocumentInternal(docId);
|
|
4254
6045
|
if (!loadedDoc) {
|
|
4255
|
-
throw new
|
|
6046
|
+
throw new errors_1.DocumentNotFoundError(docId);
|
|
4256
6047
|
}
|
|
4257
6048
|
internalDoc = loadedDoc;
|
|
4258
6049
|
this.logger.debug(`Successfully loaded document ${docId} from store for ${useCustomKey ? 'changeDocWithSigningKey' : 'changeDoc'}`);
|
|
@@ -4261,12 +6052,13 @@ class BaseMindooDB {
|
|
|
4261
6052
|
this.logger.debug(`Document ${docId} found in cache`);
|
|
4262
6053
|
}
|
|
4263
6054
|
if (internalDoc.isDeleted) {
|
|
4264
|
-
throw new
|
|
6055
|
+
throw new errors_1.DocumentDeletedError(docId);
|
|
4265
6056
|
}
|
|
4266
6057
|
// Apply the change function
|
|
4267
6058
|
const now = Date.now();
|
|
4268
6059
|
this.logger.debug(`Applying change function to document ${docId}`);
|
|
4269
|
-
|
|
6060
|
+
if (this.isDebugEnabled())
|
|
6061
|
+
this.logger.debug(`Document state before change: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(internalDoc.doc))}`);
|
|
4270
6062
|
// For async callbacks, we need to handle document modifications carefully.
|
|
4271
6063
|
// Automerge.change() requires synchronous modifications, so we'll:
|
|
4272
6064
|
// 1. Execute the async callback to do any async work (like signing)
|
|
@@ -4281,6 +6073,8 @@ class BaseMindooDB {
|
|
|
4281
6073
|
const pendingAttachmentRemovals = new Set();
|
|
4282
6074
|
// Map of attachmentId -> {lastChunkId, sizeIncrease} for appends
|
|
4283
6075
|
const pendingAttachmentAppends = new Map();
|
|
6076
|
+
// Map of attachmentId -> extraction result for setAttachmentExtractedText
|
|
6077
|
+
const pendingAttachmentTextUpdates = new Map();
|
|
4284
6078
|
// Reference to db for closures
|
|
4285
6079
|
const db = this;
|
|
4286
6080
|
// Guard flag to prevent operations after callback completes
|
|
@@ -4299,6 +6093,9 @@ class BaseMindooDB {
|
|
|
4299
6093
|
getLastModified: () => internalDoc.lastModified,
|
|
4300
6094
|
getDecryptionKeyId: () => internalDoc.decryptionKeyId,
|
|
4301
6095
|
isDeleted: () => false,
|
|
6096
|
+
isAccessible: () => true,
|
|
6097
|
+
isAwaitingWitness: () => internalDoc.awaitingWitness ?? false,
|
|
6098
|
+
isWitnessed: () => internalDoc.witnessed ?? false,
|
|
4302
6099
|
getHeads: () => automerge_adapter_1.Automerge.getHeads(internalDoc.doc),
|
|
4303
6100
|
getData: () => {
|
|
4304
6101
|
// Return a proxy that collects property assignments and deletions
|
|
@@ -4388,6 +6185,44 @@ class BaseMindooDB {
|
|
|
4388
6185
|
pendingAttachmentRemovals.add(attachmentId);
|
|
4389
6186
|
}
|
|
4390
6187
|
},
|
|
6188
|
+
/**
|
|
6189
|
+
* Store the result of an external text extraction (e.g. OCR) at the
|
|
6190
|
+
* attachment entry in `_attachments`. See the full contract on
|
|
6191
|
+
* {@link MindooDoc.setAttachmentExtractedText} in types.ts.
|
|
6192
|
+
*
|
|
6193
|
+
* Behavior inside this changeDoc draft:
|
|
6194
|
+
* - Only records a pending update here; the `_attachments` array is
|
|
6195
|
+
* mutated once in the surrounding Automerge.change block when the
|
|
6196
|
+
* callback completes, alongside pending adds/removals/appends.
|
|
6197
|
+
* - The target attachment must exist in the current document or among
|
|
6198
|
+
* attachments added earlier in this same changeDoc call; attachments
|
|
6199
|
+
* already marked for removal in this call don't count.
|
|
6200
|
+
* - `text` is capped at ATTACHMENT_EXTRACTED_TEXT_MAX_CHARS before
|
|
6201
|
+
* being persisted. `text: null` clears a previous result including
|
|
6202
|
+
* its status/engine/extractedAt metadata.
|
|
6203
|
+
* - Calling this repeatedly for the same attachmentId within one
|
|
6204
|
+
* changeDoc keeps only the last result (map semantics).
|
|
6205
|
+
*
|
|
6206
|
+
* @param attachmentId ID of an attachment of this document.
|
|
6207
|
+
* @param result Extraction outcome; `status` defaults to "done" when
|
|
6208
|
+
* text is provided (applied in the change block).
|
|
6209
|
+
* @throws Error if the attachment does not exist (or was removed in
|
|
6210
|
+
* this changeDoc call), or when invoked after the callback finished.
|
|
6211
|
+
*/
|
|
6212
|
+
setAttachmentExtractedText: (attachmentId, result) => {
|
|
6213
|
+
throwIfCallbackInactive('setAttachmentExtractedText');
|
|
6214
|
+
const payload = internalDoc.doc;
|
|
6215
|
+
const existingAttachments = payload._attachments || [];
|
|
6216
|
+
const existsInDoc = existingAttachments.some((a) => a.attachmentId === attachmentId && !pendingAttachmentRemovals.has(attachmentId));
|
|
6217
|
+
const existsInPending = pendingAttachmentAdditions.some((a) => a.attachmentId === attachmentId);
|
|
6218
|
+
if (!existsInDoc && !existsInPending) {
|
|
6219
|
+
throw new Error(`Attachment ${attachmentId} not found in document ${docId}`);
|
|
6220
|
+
}
|
|
6221
|
+
const cappedText = typeof result.text === "string" && result.text.length > types_1.ATTACHMENT_EXTRACTED_TEXT_MAX_CHARS
|
|
6222
|
+
? result.text.slice(0, types_1.ATTACHMENT_EXTRACTED_TEXT_MAX_CHARS)
|
|
6223
|
+
: result.text;
|
|
6224
|
+
pendingAttachmentTextUpdates.set(attachmentId, { ...result, text: cappedText });
|
|
6225
|
+
},
|
|
4391
6226
|
appendToAttachment: async (attachmentId, data) => {
|
|
4392
6227
|
throwIfCallbackInactive('appendToAttachment');
|
|
4393
6228
|
// Find the attachment reference
|
|
@@ -4461,7 +6296,10 @@ class BaseMindooDB {
|
|
|
4461
6296
|
delete automergeDoc[key];
|
|
4462
6297
|
}
|
|
4463
6298
|
// Apply pending attachment changes
|
|
4464
|
-
if (pendingAttachmentAdditions.length > 0 ||
|
|
6299
|
+
if (pendingAttachmentAdditions.length > 0 ||
|
|
6300
|
+
pendingAttachmentRemovals.size > 0 ||
|
|
6301
|
+
pendingAttachmentAppends.size > 0 ||
|
|
6302
|
+
pendingAttachmentTextUpdates.size > 0) {
|
|
4465
6303
|
// Initialize _attachments array if needed
|
|
4466
6304
|
if (!automergeDoc._attachments) {
|
|
4467
6305
|
automergeDoc._attachments = [];
|
|
@@ -4486,11 +6324,40 @@ class BaseMindooDB {
|
|
|
4486
6324
|
for (const ref of pendingAttachmentAdditions) {
|
|
4487
6325
|
attachments.push(ref);
|
|
4488
6326
|
}
|
|
6327
|
+
// Apply extraction results (setAttachmentExtractedText)
|
|
6328
|
+
for (const [attachmentId, update] of pendingAttachmentTextUpdates) {
|
|
6329
|
+
const attachment = attachments.find(a => a.attachmentId === attachmentId);
|
|
6330
|
+
if (!attachment) {
|
|
6331
|
+
continue; // removed in the same changeDoc call
|
|
6332
|
+
}
|
|
6333
|
+
const isFullClear = update.text === null && update.status === undefined;
|
|
6334
|
+
if (isFullClear) {
|
|
6335
|
+
// `text: null` without a status resets the entry entirely
|
|
6336
|
+
// (e.g. before a re-extraction).
|
|
6337
|
+
delete attachment.extractedText;
|
|
6338
|
+
delete attachment.extractionStatus;
|
|
6339
|
+
delete attachment.extractionEngine;
|
|
6340
|
+
delete attachment.extractedAt;
|
|
6341
|
+
continue;
|
|
6342
|
+
}
|
|
6343
|
+
if (update.text === null) {
|
|
6344
|
+
delete attachment.extractedText;
|
|
6345
|
+
}
|
|
6346
|
+
else {
|
|
6347
|
+
attachment.extractedText = update.text;
|
|
6348
|
+
}
|
|
6349
|
+
attachment.extractionStatus = update.status ?? "done";
|
|
6350
|
+
if (update.engine !== undefined) {
|
|
6351
|
+
attachment.extractionEngine = update.engine;
|
|
6352
|
+
}
|
|
6353
|
+
attachment.extractedAt = update.extractedAt ?? now;
|
|
6354
|
+
}
|
|
4489
6355
|
}
|
|
4490
6356
|
// Update lastModified timestamp
|
|
4491
6357
|
automergeDoc._lastModified = now;
|
|
4492
6358
|
}));
|
|
4493
|
-
|
|
6359
|
+
if (this.isDebugEnabled())
|
|
6360
|
+
this.logger.debug(`Successfully applied change function, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
|
|
4494
6361
|
}
|
|
4495
6362
|
catch (error) {
|
|
4496
6363
|
this.logger.error(`Error in Automerge.change for document ${docId}:`, error);
|
|
@@ -4508,6 +6375,7 @@ class BaseMindooDB {
|
|
|
4508
6375
|
signingKeyPassword,
|
|
4509
6376
|
successMessage: useCustomKey ? "changed with custom signing key" : "changed",
|
|
4510
6377
|
attachmentIds: pendingAttachmentAdditions.map((ref) => ref.attachmentId),
|
|
6378
|
+
bypassPrecheck,
|
|
4511
6379
|
});
|
|
4512
6380
|
}
|
|
4513
6381
|
catch (error) {
|
|
@@ -5002,7 +6870,7 @@ class BaseMindooDB {
|
|
|
5002
6870
|
}
|
|
5003
6871
|
}
|
|
5004
6872
|
async persistDocumentChange(options) {
|
|
5005
|
-
const { internalDoc, newDoc, now, headsBeforeChange, useCustomKey, signingKeyPair, signingKeyPassword, successMessage, attachmentIds } = options;
|
|
6873
|
+
const { internalDoc, newDoc, now, headsBeforeChange, useCustomKey, signingKeyPair, signingKeyPassword, successMessage, attachmentIds, bypassPrecheck } = options;
|
|
5006
6874
|
const docId = internalDoc.id;
|
|
5007
6875
|
this.logger.debug(`Getting change bytes from document ${docId}`);
|
|
5008
6876
|
const changesSincePreviousHeads = headsBeforeChange
|
|
@@ -5025,10 +6893,11 @@ class BaseMindooDB {
|
|
|
5025
6893
|
signingKeyPassword,
|
|
5026
6894
|
successMessage,
|
|
5027
6895
|
attachmentIds,
|
|
6896
|
+
bypassPrecheck,
|
|
5028
6897
|
});
|
|
5029
6898
|
}
|
|
5030
6899
|
async persistAutomergeDocumentChanges(options) {
|
|
5031
|
-
const { internalDoc, newDoc, now, changeBytesList, headsBeforeChange, useCustomKey, signingKeyPair, signingKeyPassword, successMessage, attachmentIds, } = options;
|
|
6900
|
+
const { internalDoc, newDoc, now, changeBytesList, headsBeforeChange, useCustomKey, signingKeyPair, signingKeyPassword, successMessage, attachmentIds, bypassPrecheck, } = options;
|
|
5032
6901
|
const docId = internalDoc.id;
|
|
5033
6902
|
const resolvedChangeBytesList = changeBytesList ?? (headsBeforeChange
|
|
5034
6903
|
? automerge_adapter_1.Automerge.getChangesSince(newDoc, headsBeforeChange)
|
|
@@ -5036,6 +6905,10 @@ class BaseMindooDB {
|
|
|
5036
6905
|
if (resolvedChangeBytesList.length === 0) {
|
|
5037
6906
|
throw new Error("No Automerge changes to persist");
|
|
5038
6907
|
}
|
|
6908
|
+
// Signed snapshot of the document's attachments after this change, computed
|
|
6909
|
+
// once from the resulting document state so EVERY change path (changeDoc,
|
|
6910
|
+
// text patches, merges, ...) carries it consistently on each produced entry.
|
|
6911
|
+
const attachmentRefs = this.collectAttachmentRefs(newDoc);
|
|
5039
6912
|
const entries = [];
|
|
5040
6913
|
for (let index = 0; index < resolvedChangeBytesList.length; index += 1) {
|
|
5041
6914
|
const changeBytes = resolvedChangeBytesList[index];
|
|
@@ -5047,16 +6920,47 @@ class BaseMindooDB {
|
|
|
5047
6920
|
signingKeyPair,
|
|
5048
6921
|
signingKeyPassword,
|
|
5049
6922
|
attachmentIds: index === 0 ? attachmentIds : undefined,
|
|
6923
|
+
// The full attachment snapshot describes the document's resulting state,
|
|
6924
|
+
// so it goes on every produced entry (unlike the add-only attachmentIds).
|
|
6925
|
+
attachmentRefs,
|
|
5050
6926
|
});
|
|
5051
6927
|
entries.push(entry);
|
|
5052
6928
|
const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
|
|
5053
6929
|
this.registerAutomergeHashMapping(docId, decodedChange.hash, entry.id);
|
|
5054
6930
|
}
|
|
6931
|
+
// Client-side write-policy precheck (docs/accesscontrol.md §9). `internalDoc.doc`
|
|
6932
|
+
// is still the pre-change state here (it is reassigned to `newDoc` below,
|
|
6933
|
+
// after the store write), so before/after are both available for Tier 2
|
|
6934
|
+
// (`withfields`) evaluation. Throws AccessDeniedError when denied.
|
|
6935
|
+
{
|
|
6936
|
+
const signerKey = useCustomKey
|
|
6937
|
+
? signingKeyPair.publicKey
|
|
6938
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
6939
|
+
const creatorKey = await this.resolveCreatorSigningKey(docId);
|
|
6940
|
+
const beforeState = internalDoc.doc;
|
|
6941
|
+
await this.assertWriteAllowed({
|
|
6942
|
+
op: "doc_change",
|
|
6943
|
+
signerKey,
|
|
6944
|
+
isAuthor: creatorKey !== null && signerKey === creatorKey,
|
|
6945
|
+
bypass: bypassPrecheck,
|
|
6946
|
+
getBeforeDoc: () => this.convertAutomergeToJS(beforeState),
|
|
6947
|
+
getAfterDoc: () => this.convertAutomergeToJS(newDoc),
|
|
6948
|
+
});
|
|
6949
|
+
}
|
|
5055
6950
|
await this.store.putEntries(entries);
|
|
5056
6951
|
internalDoc.doc = newDoc;
|
|
5057
6952
|
internalDoc.lastModified = now;
|
|
6953
|
+
// The change entries just written are versioned and un-witnessed, so the
|
|
6954
|
+
// document is awaiting witness (until pushed + stamped with a receipt).
|
|
6955
|
+
internalDoc.awaitingWitness = entries.some(storeEntryTime_1.isProvisional) || (internalDoc.awaitingWitness ?? false);
|
|
6956
|
+
// `versioned` is sticky: any versioned entry (new or already reflected by a
|
|
6957
|
+
// prior awaiting/witnessed flag) keeps the doc in the witness-aware era.
|
|
6958
|
+
const versioned = entries.some(storeEntryTime_1.isVersioned)
|
|
6959
|
+
|| internalDoc.awaitingWitness
|
|
6960
|
+
|| (internalDoc.witnessed ?? false);
|
|
6961
|
+
internalDoc.witnessed = versioned && !internalDoc.awaitingWitness;
|
|
5058
6962
|
await this.storeCachedDocument(internalDoc);
|
|
5059
|
-
this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible");
|
|
6963
|
+
this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted, internalDoc.decryptionKeyId, "visible", internalDoc.awaitingWitness, internalDoc.witnessed);
|
|
5060
6964
|
this.markDocDirty(docId);
|
|
5061
6965
|
this.logger.info(`Document ${docId} ${successMessage} successfully`);
|
|
5062
6966
|
const createdByPublicKey = useCustomKey
|
|
@@ -5067,7 +6971,7 @@ class BaseMindooDB {
|
|
|
5067
6971
|
: { createdByPublicKey: createdByPublicKey ?? "" });
|
|
5068
6972
|
}
|
|
5069
6973
|
async buildDocChangeStoreEntry(options) {
|
|
5070
|
-
const { internalDoc, changeBytes, now, useCustomKey, signingKeyPair, signingKeyPassword, attachmentIds } = options;
|
|
6974
|
+
const { internalDoc, changeBytes, now, useCustomKey, signingKeyPair, signingKeyPassword, attachmentIds, attachmentRefs } = options;
|
|
5071
6975
|
const docId = internalDoc.id;
|
|
5072
6976
|
this.logger.debug(`Got change bytes: ${changeBytes.length} bytes`);
|
|
5073
6977
|
const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
|
|
@@ -5105,7 +7009,11 @@ class BaseMindooDB {
|
|
|
5105
7009
|
originalSize: changeBytes.length,
|
|
5106
7010
|
encryptedSize: encryptedPayload.length,
|
|
5107
7011
|
attachmentIds: attachmentIds && attachmentIds.length > 0 ? attachmentIds : undefined,
|
|
7012
|
+
attachmentRefs: attachmentRefs && attachmentRefs.length > 0 ? attachmentRefs : undefined,
|
|
7013
|
+
entryVersion: types_1.CURRENT_STORE_ENTRY_VERSION,
|
|
5108
7014
|
};
|
|
7015
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
7016
|
+
entryMetadata.metadataSignature = await this.computeEntryMetadataSignature(entryMetadata, useCustomKey ? { signingKeyPair, signingKeyPassword } : undefined);
|
|
5109
7017
|
return {
|
|
5110
7018
|
...entryMetadata,
|
|
5111
7019
|
encryptedData: encryptedPayload,
|
|
@@ -5119,6 +7027,18 @@ class BaseMindooDB {
|
|
|
5119
7027
|
async maybeWriteSnapshotForDocument(internalDoc, options) {
|
|
5120
7028
|
const docId = internalDoc.id;
|
|
5121
7029
|
try {
|
|
7030
|
+
// Cheap pre-filter: while the in-memory write counter says the doc is
|
|
7031
|
+
// still below the snapshot threshold, skip the metadata scan entirely.
|
|
7032
|
+
// An absent counter (first write this session) forces one scan to learn
|
|
7033
|
+
// the real backlog, after which the counter tracks it incrementally.
|
|
7034
|
+
const knownWrites = this.writesSinceSnapshotCheck.get(docId);
|
|
7035
|
+
if (knownWrites !== undefined) {
|
|
7036
|
+
const writes = knownWrites + 1;
|
|
7037
|
+
if (writes < this.snapshotMinChanges) {
|
|
7038
|
+
this.writesSinceSnapshotCheck.set(docId, writes);
|
|
7039
|
+
return;
|
|
7040
|
+
}
|
|
7041
|
+
}
|
|
5122
7042
|
const allMetadata = await this.scanAllMetadata(this.store, { docId });
|
|
5123
7043
|
const replayEntries = allMetadata.filter((em) => this.isDocumentReplayEntry(em));
|
|
5124
7044
|
const snapshots = allMetadata
|
|
@@ -5127,6 +7047,7 @@ class BaseMindooDB {
|
|
|
5127
7047
|
const latestSnapshot = snapshots[0] || null;
|
|
5128
7048
|
const latestSnapshotAt = latestSnapshot?.createdAt ?? 0;
|
|
5129
7049
|
const changesSinceSnapshot = replayEntries.filter((em) => em.createdAt > latestSnapshotAt).length;
|
|
7050
|
+
this.writesSinceSnapshotCheck.set(docId, changesSinceSnapshot);
|
|
5130
7051
|
if (changesSinceSnapshot < this.snapshotMinChanges) {
|
|
5131
7052
|
return;
|
|
5132
7053
|
}
|
|
@@ -5148,6 +7069,9 @@ class BaseMindooDB {
|
|
|
5148
7069
|
else {
|
|
5149
7070
|
signature = await this.tenant.signPayload(encryptedPayload);
|
|
5150
7071
|
}
|
|
7072
|
+
// Carry the live attachment snapshot so the referenced set survives history
|
|
7073
|
+
// compaction (a snapshot may be the only entry a peer retains for the doc).
|
|
7074
|
+
const snapshotAttachmentRefs = this.collectAttachmentRefs(internalDoc.doc);
|
|
5151
7075
|
const snapshotEntry = {
|
|
5152
7076
|
entryType: "doc_snapshot",
|
|
5153
7077
|
id: entryId,
|
|
@@ -5162,9 +7086,15 @@ class BaseMindooDB {
|
|
|
5162
7086
|
signature,
|
|
5163
7087
|
originalSize: snapshotBytes.length,
|
|
5164
7088
|
encryptedSize: encryptedPayload.length,
|
|
7089
|
+
attachmentRefs: snapshotAttachmentRefs.length > 0 ? snapshotAttachmentRefs : undefined,
|
|
5165
7090
|
encryptedData: encryptedPayload,
|
|
5166
7091
|
};
|
|
7092
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
7093
|
+
snapshotEntry.metadataSignature = await this.computeEntryMetadataSignature(snapshotEntry, options.signingKeyPair && options.signingKeyPassword
|
|
7094
|
+
? { signingKeyPair: options.signingKeyPair, signingKeyPassword: options.signingKeyPassword }
|
|
7095
|
+
: undefined);
|
|
5167
7096
|
await this.store.putEntries([snapshotEntry]);
|
|
7097
|
+
this.writesSinceSnapshotCheck.set(docId, 0);
|
|
5168
7098
|
this.logger.debug(`Created snapshot for document ${docId} with ${headHashes.length} heads and ${changesSinceSnapshot} changes since previous snapshot`);
|
|
5169
7099
|
}
|
|
5170
7100
|
catch (error) {
|
|
@@ -5181,7 +7111,10 @@ class BaseMindooDB {
|
|
|
5181
7111
|
// materialize the entire remaining changefeed tail up front.
|
|
5182
7112
|
for (let i = startIndex; i < indexSnapshot.length && docIds.length < this.iteratePrefetchWindowDocs; i++) {
|
|
5183
7113
|
const entry = indexSnapshot[i];
|
|
5184
|
-
|
|
7114
|
+
// Skip docs that iterateChangesSince will emit as lightweight
|
|
7115
|
+
// tombstones anyway: inaccessible docs are never materialized, and
|
|
7116
|
+
// deleted docs are yielded without body materialization.
|
|
7117
|
+
if (entry.accessState === "inaccessible" || entry.isDeleted) {
|
|
5185
7118
|
continue;
|
|
5186
7119
|
}
|
|
5187
7120
|
const docId = entry.docId;
|
|
@@ -5430,7 +7363,7 @@ class BaseMindooDB {
|
|
|
5430
7363
|
this.logger.warn(`Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
|
|
5431
7364
|
}
|
|
5432
7365
|
else {
|
|
5433
|
-
isValid = await this.tenant.
|
|
7366
|
+
isValid = await this.tenant.verifyEntrySignature(snapshotData, snapshotData.encryptedData);
|
|
5434
7367
|
}
|
|
5435
7368
|
if (!isValid) {
|
|
5436
7369
|
this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to replay without snapshot`);
|
|
@@ -5439,11 +7372,19 @@ class BaseMindooDB {
|
|
|
5439
7372
|
else {
|
|
5440
7373
|
const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
|
|
5441
7374
|
doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
|
|
5442
|
-
|
|
5443
|
-
if (
|
|
5444
|
-
this.
|
|
7375
|
+
// Snapshot-head verification (docs/accesscontrol.md §10).
|
|
7376
|
+
if (!(0, materializationGuard_1.snapshotHeadsMatch)(automerge_adapter_1.Automerge.getHeads(doc), snapshotData.snapshotHeadHashes)) {
|
|
7377
|
+
this.logger.warn(`Snapshot ${snapshotData.id} heads do not match declared snapshotHeadHashes, falling back to replay without snapshot`);
|
|
7378
|
+
doc = undefined;
|
|
7379
|
+
startFromSnapshot = false;
|
|
7380
|
+
}
|
|
7381
|
+
else {
|
|
7382
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
|
|
7383
|
+
if (parsed) {
|
|
7384
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, snapshotData.id);
|
|
7385
|
+
}
|
|
7386
|
+
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
5445
7387
|
}
|
|
5446
|
-
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
5447
7388
|
}
|
|
5448
7389
|
}
|
|
5449
7390
|
}
|
|
@@ -5467,7 +7408,7 @@ class BaseMindooDB {
|
|
|
5467
7408
|
this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
|
|
5468
7409
|
continue;
|
|
5469
7410
|
}
|
|
5470
|
-
const isValid = await this.tenant.
|
|
7411
|
+
const isValid = await this.tenant.verifyEntrySignature(entryData, entryData.encryptedData);
|
|
5471
7412
|
if (!isValid) {
|
|
5472
7413
|
this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
|
|
5473
7414
|
continue;
|
|
@@ -5482,6 +7423,7 @@ class BaseMindooDB {
|
|
|
5482
7423
|
const orderedReplayEntries = [...replayEntriesForState].sort((left, right) => left.createdAt !== right.createdAt ? left.createdAt - right.createdAt : left.id.localeCompare(right.id));
|
|
5483
7424
|
const firstReplayEntry = orderedReplayEntries[0] ?? null;
|
|
5484
7425
|
const lastReplayEntry = orderedReplayEntries[orderedReplayEntries.length - 1] ?? null;
|
|
7426
|
+
const witnessState = (0, storeEntryTime_1.metadataWitnessState)(replayEntriesForState);
|
|
5485
7427
|
return {
|
|
5486
7428
|
id: docId,
|
|
5487
7429
|
doc,
|
|
@@ -5489,6 +7431,8 @@ class BaseMindooDB {
|
|
|
5489
7431
|
lastModified: lastReplayEntry?.createdAt ?? snapshotMeta?.createdAt ?? fallbackTimestamp,
|
|
5490
7432
|
decryptionKeyId: firstReplayEntry?.decryptionKeyId ?? snapshotMeta?.decryptionKeyId ?? "default",
|
|
5491
7433
|
isDeleted: this.computeIsDeletedFromMetadata(replayEntriesForState),
|
|
7434
|
+
awaitingWitness: witnessState.awaitingWitness,
|
|
7435
|
+
witnessed: witnessState.witnessed,
|
|
5492
7436
|
};
|
|
5493
7437
|
}
|
|
5494
7438
|
async *iterateChangeMetadataSince(cursor) {
|
|
@@ -5540,6 +7484,20 @@ class BaseMindooDB {
|
|
|
5540
7484
|
docId: latestEntry.docId,
|
|
5541
7485
|
};
|
|
5542
7486
|
}
|
|
7487
|
+
/**
|
|
7488
|
+
* Count the changefeed entries after the given cursor without iterating
|
|
7489
|
+
* them. O(log n) binary search on the internal `(changeSeq, docId)` index —
|
|
7490
|
+
* used for progress reporting around {@link iterateChangesSince}.
|
|
7491
|
+
*/
|
|
7492
|
+
countChangesSince(cursor) {
|
|
7493
|
+
const actualCursor = cursor ?? {
|
|
7494
|
+
changeSeq: 0,
|
|
7495
|
+
lastModified: 0,
|
|
7496
|
+
docId: "",
|
|
7497
|
+
};
|
|
7498
|
+
const startIndex = this.getStartIndexForCursor(this.index, actualCursor);
|
|
7499
|
+
return this.index.length - startIndex;
|
|
7500
|
+
}
|
|
5543
7501
|
async *iterateChangesSince(cursor) {
|
|
5544
7502
|
const startedAt = Date.now();
|
|
5545
7503
|
// Default to initial cursor if null is provided.
|
|
@@ -5560,6 +7518,37 @@ class BaseMindooDB {
|
|
|
5560
7518
|
for (let i = startIndex; i < indexSnapshot.length; i++) {
|
|
5561
7519
|
const entry = indexSnapshot[i];
|
|
5562
7520
|
if (entry.accessState !== "visible") {
|
|
7521
|
+
// The document was readable before but the current KeyBag can no
|
|
7522
|
+
// longer decrypt it (e.g. a revoked key, §13). Emit a tombstone so
|
|
7523
|
+
// incremental consumers get a removal signal — distinguishable from a
|
|
7524
|
+
// genuine deletion via isAccessible() === false (parity with
|
|
7525
|
+
// iterateChangeMetadataSince, which already emits isDeleted: true for
|
|
7526
|
+
// these). The body is never materialized.
|
|
7527
|
+
const inaccessibleCursor = {
|
|
7528
|
+
changeSeq: entry.changeSeq,
|
|
7529
|
+
lastModified: entry.lastModified,
|
|
7530
|
+
docId: entry.docId,
|
|
7531
|
+
};
|
|
7532
|
+
yieldedDocuments++;
|
|
7533
|
+
yield { doc: this.buildInaccessibleDoc(entry), cursor: inaccessibleCursor };
|
|
7534
|
+
prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
|
|
7535
|
+
continue;
|
|
7536
|
+
}
|
|
7537
|
+
if (entry.isDeleted) {
|
|
7538
|
+
// Deleted documents are emitted as lightweight tombstones without
|
|
7539
|
+
// materializing the body — consistent with the inaccessible case
|
|
7540
|
+
// above (isDeleted() === true, getData() === {}), distinguishable
|
|
7541
|
+
// via isAccessible() === true. Consumers that need the last state
|
|
7542
|
+
// of a deleted doc can use iterateDocumentHistory /
|
|
7543
|
+
// getDocumentAtTimestamp.
|
|
7544
|
+
const deletedCursor = {
|
|
7545
|
+
changeSeq: entry.changeSeq,
|
|
7546
|
+
lastModified: entry.lastModified,
|
|
7547
|
+
docId: entry.docId,
|
|
7548
|
+
};
|
|
7549
|
+
yieldedDocuments++;
|
|
7550
|
+
yield { doc: this.buildDeletedDoc(entry), cursor: deletedCursor };
|
|
7551
|
+
prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
|
|
5563
7552
|
continue;
|
|
5564
7553
|
}
|
|
5565
7554
|
try {
|
|
@@ -5675,7 +7664,7 @@ class BaseMindooDB {
|
|
|
5675
7664
|
// Key was not trusted or failed to import
|
|
5676
7665
|
return { entryData, isValid: false };
|
|
5677
7666
|
}
|
|
5678
|
-
const isValid = await this.
|
|
7667
|
+
const isValid = await this.verifyEntrySignatureWithKey(cryptoKey, entryData);
|
|
5679
7668
|
return { entryData, isValid };
|
|
5680
7669
|
}));
|
|
5681
7670
|
// Filter out entries with invalid signatures
|
|
@@ -5715,17 +7704,29 @@ class BaseMindooDB {
|
|
|
5715
7704
|
// Check if document actually changed
|
|
5716
7705
|
const headsAfter = automerge_adapter_1.Automerge.getHeads(updatedDoc);
|
|
5717
7706
|
const headsChanged = JSON.stringify(headsBefore) !== JSON.stringify(headsAfter);
|
|
5718
|
-
|
|
5719
|
-
|
|
7707
|
+
// A witness receipt changes only an entry's `receivedAt` (metadata), not the
|
|
7708
|
+
// Automerge content, so a freshly-witnessed entry re-delivered by the cursor
|
|
7709
|
+
// scan leaves the heads UNCHANGED. We must still recompute the per-doc
|
|
7710
|
+
// "awaiting witness" flag and refresh the index/cache so the doc is
|
|
7711
|
+
// re-delivered on the metadata feed and re-evaluated by VirtualViews;
|
|
7712
|
+
// otherwise the flag would never clear. So we compute it up front (reusing
|
|
7713
|
+
// the metadata scan we need anyway) and only bail when NOTHING observable
|
|
7714
|
+
// changed — neither the content nor the witness status.
|
|
7715
|
+
const allDocMetadata = await this.scanAllMetadata(this.store, { docId });
|
|
7716
|
+
const { awaitingWitness, witnessed } = (0, storeEntryTime_1.metadataWitnessState)(allDocMetadata);
|
|
7717
|
+
const awaitingWitnessChanged = awaitingWitness !== (cachedDoc.awaitingWitness ?? false);
|
|
7718
|
+
const witnessedChanged = witnessed !== (cachedDoc.witnessed ?? false);
|
|
7719
|
+
if (!headsChanged && !awaitingWitnessChanged && !witnessedChanged) {
|
|
7720
|
+
this.logger.debug(`Document ${docId} heads and witness status unchanged after applying new entries`);
|
|
5720
7721
|
return null; // Document didn't actually change
|
|
5721
7722
|
}
|
|
5722
|
-
|
|
7723
|
+
if (this.isDebugEnabled())
|
|
7724
|
+
this.logger.debug(`Document ${docId} changed: heads before=${JSON.stringify(headsBefore)}, after=${JSON.stringify(headsAfter)}, awaitingWitness=${awaitingWitness}`);
|
|
5723
7725
|
// Update metadata
|
|
5724
7726
|
const payload = updatedDoc;
|
|
5725
7727
|
const lastEntryCreatedAt = entries.reduce((maxCreatedAt, entry) => Math.max(maxCreatedAt, entry.createdAt), cachedDoc.lastModified);
|
|
5726
7728
|
const lastModified = payload._lastModified ||
|
|
5727
7729
|
lastEntryCreatedAt;
|
|
5728
|
-
const allDocMetadata = await this.scanAllMetadata(this.store, { docId });
|
|
5729
7730
|
const isDeleted = this.computeIsDeletedFromMetadata(allDocMetadata);
|
|
5730
7731
|
const updatedInternalDoc = {
|
|
5731
7732
|
id: docId,
|
|
@@ -5734,6 +7735,8 @@ class BaseMindooDB {
|
|
|
5734
7735
|
lastModified,
|
|
5735
7736
|
decryptionKeyId: cachedDoc.decryptionKeyId,
|
|
5736
7737
|
isDeleted,
|
|
7738
|
+
awaitingWitness,
|
|
7739
|
+
witnessed,
|
|
5737
7740
|
};
|
|
5738
7741
|
// Update cache
|
|
5739
7742
|
await this.storeCachedDocument(updatedInternalDoc);
|
|
@@ -5791,6 +7794,7 @@ class BaseMindooDB {
|
|
|
5791
7794
|
this.logger.warn(`L2 deserialize failed for ${docId}, evicting stale record: ${e}`);
|
|
5792
7795
|
try {
|
|
5793
7796
|
await store.delete("doc", key);
|
|
7797
|
+
this.lastFlushedDocState.delete(docId);
|
|
5794
7798
|
}
|
|
5795
7799
|
catch (deleteError) {
|
|
5796
7800
|
this.logger.warn(`Failed to evict corrupt L2 record for ${docId}: ${deleteError}`);
|
|
@@ -5810,13 +7814,14 @@ class BaseMindooDB {
|
|
|
5810
7814
|
await this.purgeMaterializedDocument(docId);
|
|
5811
7815
|
return null;
|
|
5812
7816
|
}
|
|
5813
|
-
const indexEntryIdx = this.
|
|
7817
|
+
const indexEntryIdx = this.getDocIndexPosition(docId);
|
|
5814
7818
|
if (indexEntryIdx === undefined) {
|
|
5815
7819
|
// Doc isn't in the in-memory changefeed index. Either the cache is
|
|
5816
7820
|
// ahead of the index (very rare race during initial restore) or the
|
|
5817
7821
|
// L2 record is orphaned. Drop and fall through.
|
|
5818
7822
|
try {
|
|
5819
7823
|
await store.delete("doc", key);
|
|
7824
|
+
this.lastFlushedDocState.delete(docId);
|
|
5820
7825
|
}
|
|
5821
7826
|
catch (deleteError) {
|
|
5822
7827
|
this.logger.warn(`Failed to evict orphaned L2 record for ${docId}: ${deleteError}`);
|
|
@@ -5826,6 +7831,9 @@ class BaseMindooDB {
|
|
|
5826
7831
|
const currentChangeSeq = this.index[indexEntryIdx].changeSeq;
|
|
5827
7832
|
// Fresh hit: index agrees with the persisted state.
|
|
5828
7833
|
if (persistedChangeSeq === currentChangeSeq) {
|
|
7834
|
+
// Seed the flush-skip fingerprint with the state just read from L2 so
|
|
7835
|
+
// the next periodic flush doesn't rewrite an identical record.
|
|
7836
|
+
this.lastFlushedDocState.set(docId, `${persistedChangeSeq}:${automerge_adapter_1.Automerge.getHeads(internal.doc).join(",")}`);
|
|
5829
7837
|
await this.storeCachedDocument(internal);
|
|
5830
7838
|
return internal;
|
|
5831
7839
|
}
|
|
@@ -6086,6 +8094,491 @@ class BaseMindooDB {
|
|
|
6086
8094
|
emitProgress({ processed, total, phase: "cancelled" });
|
|
6087
8095
|
}
|
|
6088
8096
|
}
|
|
8097
|
+
/**
|
|
8098
|
+
* The per-tenant quarantine/audit log: entries rejected during
|
|
8099
|
+
* materialization by the access-control layer (docs/accesscontrol.md §10).
|
|
8100
|
+
* Exposed for audit views (e.g. Haven). Entries remain in the append-only
|
|
8101
|
+
* store but are excluded from materialized state and queries.
|
|
8102
|
+
*/
|
|
8103
|
+
getQuarantineLog() {
|
|
8104
|
+
return this.quarantineLog;
|
|
8105
|
+
}
|
|
8106
|
+
/**
|
|
8107
|
+
* Whether access control is active for this tenant and should gate
|
|
8108
|
+
* materialization. Admin-only databases (incl. the directory itself) are
|
|
8109
|
+
* always exempt — admin is the root of trust, and exempting the directory DB
|
|
8110
|
+
* also avoids re-entrancy. Cached briefly to keep the load fast-path cheap.
|
|
8111
|
+
* Fails closed-to-disabled on any directory error so document loads never
|
|
8112
|
+
* break due to a transient directory issue (Tier 2 only gates honest clients).
|
|
8113
|
+
*/
|
|
8114
|
+
async isAclEnforced() {
|
|
8115
|
+
if (this._isAdminOnlyDb)
|
|
8116
|
+
return false;
|
|
8117
|
+
const now = Date.now();
|
|
8118
|
+
if (this.aclActiveCache && now - this.aclActiveCache.at < 3000) {
|
|
8119
|
+
return this.aclActiveCache.value;
|
|
8120
|
+
}
|
|
8121
|
+
try {
|
|
8122
|
+
const directory = await this.tenant.openDirectory();
|
|
8123
|
+
let value = false;
|
|
8124
|
+
if (typeof directory.isAccessControlActive === "function") {
|
|
8125
|
+
value = await directory.isAccessControlActive();
|
|
8126
|
+
}
|
|
8127
|
+
this.aclActiveCache = { value, at: now };
|
|
8128
|
+
return value;
|
|
8129
|
+
}
|
|
8130
|
+
catch (error) {
|
|
8131
|
+
// Fail closed (audit finding #2): a transient directory error must never
|
|
8132
|
+
// silently DISABLE access control (which would bypass the fail-closed
|
|
8133
|
+
// materialization gate entirely). Reuse the last known verdict if we have
|
|
8134
|
+
// one; otherwise assume enforced so the materialization path governs the
|
|
8135
|
+
// load and retries once the directory is reachable again.
|
|
8136
|
+
if (this.aclActiveCache) {
|
|
8137
|
+
this.logger.debug(`[ACL] active-check failed, reusing last known verdict (${this.aclActiveCache.value}): ${error}`);
|
|
8138
|
+
return this.aclActiveCache.value;
|
|
8139
|
+
}
|
|
8140
|
+
this.logger.warn(`[ACL] active-check failed with no prior verdict, assuming enforced (fail closed): ${error}`);
|
|
8141
|
+
return true;
|
|
8142
|
+
}
|
|
8143
|
+
}
|
|
8144
|
+
/**
|
|
8145
|
+
* Synchronous client-side write-policy precheck (docs/accesscontrol.md §9).
|
|
8146
|
+
* Evaluates the full Tier 1 + Tier 2 ruleset for the candidate write and
|
|
8147
|
+
* throws {@link AccessDeniedError} when it is denied, so applications get
|
|
8148
|
+
* immediate, meaningful feedback at the call site instead of an optimistic
|
|
8149
|
+
* local write that is later rejected by the server witness or quarantined on
|
|
8150
|
+
* materialization.
|
|
8151
|
+
*
|
|
8152
|
+
* Fails CLOSED on any directory/infra error while access control is enforced
|
|
8153
|
+
* (audit finding #2): the precheck denies the write rather than letting it
|
|
8154
|
+
* through optimistically. The server witness (Tier 1) and
|
|
8155
|
+
* quarantine-on-materialization (Tier 2) remain the authoritative enforcers,
|
|
8156
|
+
* and the user can retry once the directory is reachable again.
|
|
8157
|
+
*
|
|
8158
|
+
* `getBeforeDoc`/`getAfterDoc` are lazy so the (potentially costly) Automerge
|
|
8159
|
+
* -> JS materialization is skipped entirely when the active policy has no
|
|
8160
|
+
* Tier 2 (`withfields`) content rule for this operation/database.
|
|
8161
|
+
*/
|
|
8162
|
+
async assertWriteAllowed(input) {
|
|
8163
|
+
if (input.bypass)
|
|
8164
|
+
return;
|
|
8165
|
+
const decision = await this.evaluateClientWriteAccess(input);
|
|
8166
|
+
if (decision && !decision.allowed) {
|
|
8167
|
+
throw new AccessDeniedError_1.AccessDeniedError(input.op, this.store.getId(), decision);
|
|
8168
|
+
}
|
|
8169
|
+
}
|
|
8170
|
+
/**
|
|
8171
|
+
* Resolve the `decryptionKeyId` for a `doc_create`: the caller's explicit key
|
|
8172
|
+
* when given, else the policy-configured default for this database (see
|
|
8173
|
+
* `getEffectiveDefaultCreateKeyId` on the directory), else the legacy
|
|
8174
|
+
* `"default"`. The policy default is a create-time convenience only; read/write
|
|
8175
|
+
* access is governed by key possession (distribution + rotation, §13), not the
|
|
8176
|
+
* server. Directory ACL writes always pass an explicit key, so this never
|
|
8177
|
+
* re-enters the directory for them.
|
|
8178
|
+
*/
|
|
8179
|
+
async resolveCreateKeyId(options) {
|
|
8180
|
+
if (options.decryptionKeyId !== undefined)
|
|
8181
|
+
return options.decryptionKeyId;
|
|
8182
|
+
try {
|
|
8183
|
+
const directory = await this.tenant.openDirectory();
|
|
8184
|
+
const getEffectiveDefaultCreateKeyId = directory.getEffectiveDefaultCreateKeyId?.bind(directory);
|
|
8185
|
+
if (getEffectiveDefaultCreateKeyId) {
|
|
8186
|
+
const fromPolicy = await getEffectiveDefaultCreateKeyId(this.store.getId());
|
|
8187
|
+
if (fromPolicy !== undefined)
|
|
8188
|
+
return fromPolicy;
|
|
8189
|
+
}
|
|
8190
|
+
}
|
|
8191
|
+
catch (error) {
|
|
8192
|
+
this.logger.debug(`[ACL] default create-key lookup skipped (directory error): ${error}`);
|
|
8193
|
+
}
|
|
8194
|
+
return "default";
|
|
8195
|
+
}
|
|
8196
|
+
/**
|
|
8197
|
+
* Evaluate the full client-side (Tier 1 + Tier 2) write ruleset for a
|
|
8198
|
+
* candidate write, returning the {@link AccessDecision}. Returns `null` only
|
|
8199
|
+
* when access control is not enforced for this database (callers treat `null`
|
|
8200
|
+
* as "allowed"). When access control IS enforced but the directory cannot be
|
|
8201
|
+
* consulted, it returns a DENY decision (fail closed, audit finding #2). The
|
|
8202
|
+
* `getBeforeDoc`/`getAfterDoc` materializers are only invoked when the active
|
|
8203
|
+
* policy has a Tier 2 (`withfields`) content rule for this op/db.
|
|
8204
|
+
*/
|
|
8205
|
+
async evaluateClientWriteAccess(input) {
|
|
8206
|
+
if (!(await this.isAclEnforced()))
|
|
8207
|
+
return null;
|
|
8208
|
+
const dbid = this.store.getId();
|
|
8209
|
+
try {
|
|
8210
|
+
const directory = await this.tenant.openDirectory();
|
|
8211
|
+
const evaluateClientAccess = directory.evaluateClientAccess?.bind(directory);
|
|
8212
|
+
if (!evaluateClientAccess)
|
|
8213
|
+
return null;
|
|
8214
|
+
// Only materialize the before/after documents when a content rule needs
|
|
8215
|
+
// them; pure Tier 1 (identity/op) checks never read the doc.
|
|
8216
|
+
let needContent = true;
|
|
8217
|
+
if (typeof directory.hasWriteContentRules === "function") {
|
|
8218
|
+
needContent = await directory.hasWriteContentRules(input.op, dbid);
|
|
8219
|
+
}
|
|
8220
|
+
const beforeDoc = needContent && input.getBeforeDoc ? input.getBeforeDoc() : null;
|
|
8221
|
+
const afterDoc = needContent && input.getAfterDoc ? input.getAfterDoc() : null;
|
|
8222
|
+
return await evaluateClientAccess({
|
|
8223
|
+
op: input.op,
|
|
8224
|
+
dbid,
|
|
8225
|
+
signingKey: input.signerKey,
|
|
8226
|
+
trustedTime: Date.now(),
|
|
8227
|
+
isAuthor: input.isAuthor,
|
|
8228
|
+
beforeDoc,
|
|
8229
|
+
afterDoc,
|
|
8230
|
+
});
|
|
8231
|
+
}
|
|
8232
|
+
catch (error) {
|
|
8233
|
+
// Fail closed (audit finding #2): access control IS enforced here (checked
|
|
8234
|
+
// above), so a directory error during the write precheck must not let the
|
|
8235
|
+
// write through optimistically. Deny now; the user can retry once the
|
|
8236
|
+
// directory is reachable again. The server witness remains the
|
|
8237
|
+
// authoritative backstop.
|
|
8238
|
+
this.logger.warn(`[ACL] write precheck failed, denying (fail closed): ${error}`);
|
|
8239
|
+
return {
|
|
8240
|
+
allowed: false,
|
|
8241
|
+
reason: "directory unavailable during write precheck; failing closed",
|
|
8242
|
+
tier: "tier1",
|
|
8243
|
+
};
|
|
8244
|
+
}
|
|
8245
|
+
}
|
|
8246
|
+
/**
|
|
8247
|
+
* Predict whether a `createDocument` would be allowed by the active write
|
|
8248
|
+
* policy, without writing anything (docs/accesscontrol.md §9). Lets a UI
|
|
8249
|
+
* disable a Save action and surface `decision.reason` up front. The returned
|
|
8250
|
+
* decision mirrors what {@link createDocument} would enforce.
|
|
8251
|
+
*/
|
|
8252
|
+
async canCreate(options = {}) {
|
|
8253
|
+
const keyId = await this.resolveCreateKeyId(options);
|
|
8254
|
+
const useCustomSigningKey = options.signingKeyPair !== undefined && options.signingKeyPassword !== undefined;
|
|
8255
|
+
const signerKey = useCustomSigningKey
|
|
8256
|
+
? options.signingKeyPair.publicKey
|
|
8257
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
8258
|
+
const afterValues = { _attachments: [] };
|
|
8259
|
+
if (options.initialValues) {
|
|
8260
|
+
for (const [k, v] of Object.entries(options.initialValues)) {
|
|
8261
|
+
if (!k.startsWith("_"))
|
|
8262
|
+
afterValues[k] = v;
|
|
8263
|
+
}
|
|
8264
|
+
}
|
|
8265
|
+
const decision = await this.evaluateClientWriteAccess({
|
|
8266
|
+
op: "doc_create",
|
|
8267
|
+
signerKey,
|
|
8268
|
+
isAuthor: true,
|
|
8269
|
+
getAfterDoc: () => afterValues,
|
|
8270
|
+
});
|
|
8271
|
+
return decision ?? BaseMindooDB.ACL_NOT_ENFORCED_DECISION;
|
|
8272
|
+
}
|
|
8273
|
+
/**
|
|
8274
|
+
* Predict whether applying `candidateAfter` as the document's next state via
|
|
8275
|
+
* {@link changeDoc} would be allowed by the active write policy, without
|
|
8276
|
+
* writing anything. `candidateAfter` is the intended full "after" document
|
|
8277
|
+
* (e.g. the edited JSON in a database-browser editor).
|
|
8278
|
+
*/
|
|
8279
|
+
async canChange(doc, candidateAfter, signingKeyPair) {
|
|
8280
|
+
const docId = doc.getId();
|
|
8281
|
+
const signerKey = signingKeyPair
|
|
8282
|
+
? signingKeyPair.publicKey
|
|
8283
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
8284
|
+
const creatorKey = await this.resolveCreatorSigningKey(docId);
|
|
8285
|
+
const before = doc.getData();
|
|
8286
|
+
const decision = await this.evaluateClientWriteAccess({
|
|
8287
|
+
op: "doc_change",
|
|
8288
|
+
signerKey,
|
|
8289
|
+
isAuthor: creatorKey !== null && signerKey === creatorKey,
|
|
8290
|
+
getBeforeDoc: () => before,
|
|
8291
|
+
getAfterDoc: () => candidateAfter,
|
|
8292
|
+
});
|
|
8293
|
+
return decision ?? BaseMindooDB.ACL_NOT_ENFORCED_DECISION;
|
|
8294
|
+
}
|
|
8295
|
+
/**
|
|
8296
|
+
* Predict whether deleting `doc` via {@link deleteDocument} would be allowed
|
|
8297
|
+
* by the active write policy, without writing anything.
|
|
8298
|
+
*/
|
|
8299
|
+
async canDelete(doc, signingKeyPair) {
|
|
8300
|
+
const docId = doc.getId();
|
|
8301
|
+
const signerKey = signingKeyPair
|
|
8302
|
+
? signingKeyPair.publicKey
|
|
8303
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
8304
|
+
const creatorKey = await this.resolveCreatorSigningKey(docId);
|
|
8305
|
+
const current = doc.getData();
|
|
8306
|
+
const decision = await this.evaluateClientWriteAccess({
|
|
8307
|
+
op: "doc_delete",
|
|
8308
|
+
signerKey,
|
|
8309
|
+
isAuthor: creatorKey !== null && signerKey === creatorKey,
|
|
8310
|
+
getBeforeDoc: () => current,
|
|
8311
|
+
getAfterDoc: () => current,
|
|
8312
|
+
});
|
|
8313
|
+
return decision ?? BaseMindooDB.ACL_NOT_ENFORCED_DECISION;
|
|
8314
|
+
}
|
|
8315
|
+
/**
|
|
8316
|
+
* Predict whether undeleting `doc` via {@link undeleteDocument} would be
|
|
8317
|
+
* allowed by the active write policy, without writing anything.
|
|
8318
|
+
*/
|
|
8319
|
+
async canUndelete(doc, signingKeyPair) {
|
|
8320
|
+
const docId = doc.getId();
|
|
8321
|
+
const signerKey = signingKeyPair
|
|
8322
|
+
? signingKeyPair.publicKey
|
|
8323
|
+
: (await this.tenant.getCurrentUserId()).userSigningPublicKey;
|
|
8324
|
+
const creatorKey = await this.resolveCreatorSigningKey(docId);
|
|
8325
|
+
const current = doc.getData();
|
|
8326
|
+
const decision = await this.evaluateClientWriteAccess({
|
|
8327
|
+
op: "doc_undelete",
|
|
8328
|
+
signerKey,
|
|
8329
|
+
isAuthor: creatorKey !== null && signerKey === creatorKey,
|
|
8330
|
+
getBeforeDoc: () => current,
|
|
8331
|
+
getAfterDoc: () => current,
|
|
8332
|
+
});
|
|
8333
|
+
return decision ?? BaseMindooDB.ACL_NOT_ENFORCED_DECISION;
|
|
8334
|
+
}
|
|
8335
|
+
/**
|
|
8336
|
+
* Resolve the signing public key of the document's original creator (the
|
|
8337
|
+
* `doc_create` entry's `createdByPublicKey`), used to set `$author`
|
|
8338
|
+
* (`isAuthor`) for the client write prechecks on change/delete/undelete.
|
|
8339
|
+
* Cached per document; returns `null` if the create entry isn't present yet
|
|
8340
|
+
* (in which case `isAuthor` is treated as false, matching materialization).
|
|
8341
|
+
*/
|
|
8342
|
+
async resolveCreatorSigningKey(docId) {
|
|
8343
|
+
const cached = this.creatorKeyCache.get(docId);
|
|
8344
|
+
if (cached)
|
|
8345
|
+
return cached;
|
|
8346
|
+
try {
|
|
8347
|
+
const metas = await this.scanAllMetadata(this.store, { docId });
|
|
8348
|
+
const createKey = metas.find((m) => m.entryType === "doc_create")?.createdByPublicKey ?? null;
|
|
8349
|
+
if (createKey)
|
|
8350
|
+
this.creatorKeyCache.set(docId, createKey);
|
|
8351
|
+
return createKey;
|
|
8352
|
+
}
|
|
8353
|
+
catch (error) {
|
|
8354
|
+
this.logger.debug(`[ACL] creator-key resolution failed for ${docId}: ${error}`);
|
|
8355
|
+
return null;
|
|
8356
|
+
}
|
|
8357
|
+
}
|
|
8358
|
+
/** Append a record to the quarantine/audit log (and log it). */
|
|
8359
|
+
recordQuarantine(rec) {
|
|
8360
|
+
this.quarantineLog.push(rec);
|
|
8361
|
+
console.log(`[MindooDB][ACL] Quarantined entry ${rec.entryId} (doc ${rec.docId}): ${rec.reason} — ${rec.detail}`);
|
|
8362
|
+
}
|
|
8363
|
+
/**
|
|
8364
|
+
* Materialize a document through the access-control (Tier 2) path
|
|
8365
|
+
* (docs/accesscontrol.md §10). Entries are evaluated deterministically in
|
|
8366
|
+
* trusted-time order; an entry is quarantined if it violates a rule, fails
|
|
8367
|
+
* op-type re-derivation, or causally depends on a quarantined entry
|
|
8368
|
+
* (cascade). The accepted set is therefore a causally-closed prefix identical
|
|
8369
|
+
* across replicas, which is what makes "quarantine on receipt" safe under
|
|
8370
|
+
* eventual consistency.
|
|
8371
|
+
*
|
|
8372
|
+
* Snapshots are intentionally bypassed here and the document is rebuilt from
|
|
8373
|
+
* individually-verified lifecycle entries (the spec's fallback), so a snapshot
|
|
8374
|
+
* can never smuggle content past Tier 2.
|
|
8375
|
+
*
|
|
8376
|
+
* @returns the materialized {@link InternalDoc} (or `null` if nothing is
|
|
8377
|
+
* accepted / the document is absent), or `undefined` when access control is
|
|
8378
|
+
* not enforced (caller uses the optimized fast path).
|
|
8379
|
+
*/
|
|
8380
|
+
async maybeMaterializeWithAccessControl(docId, allEntryMetadata) {
|
|
8381
|
+
if (!(await this.isAclEnforced()))
|
|
8382
|
+
return undefined;
|
|
8383
|
+
// Fail-closed self-healing (audit finding #2): when a validation step cannot
|
|
8384
|
+
// be completed (directory/witness adapter threw), we quarantine the affected
|
|
8385
|
+
// entry for THIS pass but must not cache the result, so the next load retries
|
|
8386
|
+
// and recovers once the dependency is healthy again. `cacheable` is flipped
|
|
8387
|
+
// to false the moment any transient failure occurs.
|
|
8388
|
+
let cacheable = true;
|
|
8389
|
+
const dbid = this.store.getId();
|
|
8390
|
+
const directory = await this.tenant.openDirectory();
|
|
8391
|
+
// Capture bound locals: these optional members are called later inside an
|
|
8392
|
+
// await-heavy loop, where TS would otherwise lose the property narrowing.
|
|
8393
|
+
const evaluateClientAccess = directory.evaluateClientAccess?.bind(directory);
|
|
8394
|
+
if (!evaluateClientAccess)
|
|
8395
|
+
return undefined;
|
|
8396
|
+
const getTrustedWitnessKeysAt = directory.getTrustedWitnessKeysAt?.bind(directory);
|
|
8397
|
+
// One verifier per materialization enforces per-witness `receivedAt`
|
|
8398
|
+
// monotonicity across the entries we accept, in trusted-time order (§5.4).
|
|
8399
|
+
// A verify-only provider (no signer) routes receipts to the right scheme.
|
|
8400
|
+
const subtle = this.getSubtle();
|
|
8401
|
+
const receiptVerifier = new Ed25519WitnessProvider_1.Ed25519WitnessProvider({ subtle }).createVerifier();
|
|
8402
|
+
const nowMs = Date.now();
|
|
8403
|
+
// Lifecycle entries only, folded in deterministic causal order. Un-witnessed
|
|
8404
|
+
// entries (versioned or legacy) order by their author time `createdAt`, NOT
|
|
8405
|
+
// the provisional `now`: this fold computes per-step before/after states for
|
|
8406
|
+
// Tier 2 evaluation, so it must preserve authoring order rather than
|
|
8407
|
+
// collapsing every still-provisional entry onto a single `now` (which would
|
|
8408
|
+
// tie-break by entry id and scramble causal sequence). The version-aware
|
|
8409
|
+
// head/`now` selection only matters for the changefeed stamp and the
|
|
8410
|
+
// judgment-site node selection below, not for this fold order.
|
|
8411
|
+
const dataEntries = allEntryMetadata
|
|
8412
|
+
.filter((m) => this.isDocumentReplayEntry(m))
|
|
8413
|
+
.sort((a, b) => (a.receivedAt ?? a.createdAt) === (b.receivedAt ?? b.createdAt)
|
|
8414
|
+
? a.id.localeCompare(b.id)
|
|
8415
|
+
: (a.receivedAt ?? a.createdAt) - (b.receivedAt ?? b.createdAt));
|
|
8416
|
+
if (dataEntries.length === 0)
|
|
8417
|
+
return { doc: null, cacheable };
|
|
8418
|
+
const creatorKey = dataEntries.find((m) => m.entryType === "doc_create")?.createdByPublicKey ?? null;
|
|
8419
|
+
const fetched = await this.store.getEntries(dataEntries.map((m) => m.id));
|
|
8420
|
+
const entryById = new Map(fetched.map((e) => [e.id, e]));
|
|
8421
|
+
let currentDoc = null;
|
|
8422
|
+
let createdAt = null;
|
|
8423
|
+
let decryptionKeyId = "default";
|
|
8424
|
+
const acceptedMeta = [];
|
|
8425
|
+
const quarantined = new Set();
|
|
8426
|
+
const quarantine = (meta, reason, detail) => {
|
|
8427
|
+
quarantined.add(meta.id);
|
|
8428
|
+
this.recordQuarantine({
|
|
8429
|
+
entryId: meta.id,
|
|
8430
|
+
docId,
|
|
8431
|
+
dbid,
|
|
8432
|
+
entryType: meta.entryType,
|
|
8433
|
+
reason,
|
|
8434
|
+
detail,
|
|
8435
|
+
// Audit record: the entry's own (stable) time — receipt time if
|
|
8436
|
+
// witnessed, else author time. Not the provisional head `now`.
|
|
8437
|
+
trustedTime: meta.receivedAt ?? meta.createdAt,
|
|
8438
|
+
recordedAt: Date.now(),
|
|
8439
|
+
});
|
|
8440
|
+
};
|
|
8441
|
+
for (const meta of dataEntries) {
|
|
8442
|
+
const entry = entryById.get(meta.id);
|
|
8443
|
+
if (!entry)
|
|
8444
|
+
continue;
|
|
8445
|
+
// Admin-only DBs are exempt (handled by isAclEnforced); here we honor the
|
|
8446
|
+
// tenant admin key as always-trusted via the directory identity set.
|
|
8447
|
+
// Cascade: any quarantined dependency taints this entry (§10).
|
|
8448
|
+
if (meta.dependencyIds.some((dep) => quarantined.has(dep))) {
|
|
8449
|
+
quarantine(meta, "cascade_dependent", "causally depends on a quarantined entry");
|
|
8450
|
+
continue;
|
|
8451
|
+
}
|
|
8452
|
+
// Verify the author signature over the encrypted payload.
|
|
8453
|
+
const cryptoKey = await this.getOrImportPublicKey(entry.createdByPublicKey);
|
|
8454
|
+
if (!cryptoKey || !(await this.verifyEntrySignatureWithKey(cryptoKey, entry))) {
|
|
8455
|
+
quarantine(meta, "invalid_signature", "signature verification failed");
|
|
8456
|
+
continue;
|
|
8457
|
+
}
|
|
8458
|
+
// Validate the witness receipt, if any (§5.4): a present receipt must come
|
|
8459
|
+
// from a currently-trusted witness, carry a valid signature, and respect
|
|
8460
|
+
// per-witness monotonicity and the future-time bound. Entries with no
|
|
8461
|
+
// receipt are local/not-yet-witnessed and pass this check.
|
|
8462
|
+
if (getTrustedWitnessKeysAt && (meta.receivedAt !== undefined || meta.receivedByPublicKey)) {
|
|
8463
|
+
try {
|
|
8464
|
+
const trustedWitnessKeys = await getTrustedWitnessKeysAt(meta.receivedAt ?? meta.createdAt);
|
|
8465
|
+
const receiptResult = await receiptVerifier.validate(meta, { dbid, trustedWitnessKeys, nowMs });
|
|
8466
|
+
if (!receiptResult.ok) {
|
|
8467
|
+
quarantine(meta, "invalid_witness_receipt", receiptResult.reason ?? "invalid witness receipt");
|
|
8468
|
+
continue;
|
|
8469
|
+
}
|
|
8470
|
+
}
|
|
8471
|
+
catch (error) {
|
|
8472
|
+
// Fail closed (audit finding #2): a malformed receipt or adapter bug
|
|
8473
|
+
// must NOT bypass the witness check. Quarantine this entry for now and
|
|
8474
|
+
// mark the result non-cacheable so a later load retries once the
|
|
8475
|
+
// verifier is healthy again.
|
|
8476
|
+
this.logger.warn(`[ACL] receipt validation failed for ${meta.id}, quarantining (will retry): ${error}`);
|
|
8477
|
+
cacheable = false;
|
|
8478
|
+
quarantine(meta, "directory_unavailable", `witness receipt validation error: ${error}`);
|
|
8479
|
+
continue;
|
|
8480
|
+
}
|
|
8481
|
+
}
|
|
8482
|
+
const isGenesis = currentDoc === null;
|
|
8483
|
+
if (isGenesis && meta.entryType !== "doc_create") {
|
|
8484
|
+
// Not a valid document start (mirrors the fast path); skip.
|
|
8485
|
+
continue;
|
|
8486
|
+
}
|
|
8487
|
+
// Op-type defense in depth (§10): a doc_create must be a genesis change.
|
|
8488
|
+
if (meta.entryType === "doc_create" && !isGenesis) {
|
|
8489
|
+
quarantine(meta, "op_type_mismatch", "signed doc_create but not a genesis change");
|
|
8490
|
+
continue;
|
|
8491
|
+
}
|
|
8492
|
+
// Decrypt and compute the candidate (after) state.
|
|
8493
|
+
const decrypted = await this.tenant.decryptPayload(entry.encryptedData, entry.decryptionKeyId);
|
|
8494
|
+
const baseDoc = currentDoc ?? automerge_adapter_1.Automerge.init();
|
|
8495
|
+
let afterDoc;
|
|
8496
|
+
try {
|
|
8497
|
+
afterDoc = automerge_adapter_1.Automerge.loadIncremental(automerge_adapter_1.Automerge.clone(baseDoc), decrypted);
|
|
8498
|
+
}
|
|
8499
|
+
catch (error) {
|
|
8500
|
+
quarantine(meta, "op_type_mismatch", `failed to decode change: ${error}`);
|
|
8501
|
+
continue;
|
|
8502
|
+
}
|
|
8503
|
+
const beforeJS = currentDoc
|
|
8504
|
+
? this.convertAutomergeToJS(currentDoc)
|
|
8505
|
+
: null;
|
|
8506
|
+
const afterJS = this.convertAutomergeToJS(afterDoc);
|
|
8507
|
+
const isAuthor = creatorKey !== null && entry.createdByPublicKey === creatorKey;
|
|
8508
|
+
// Trusted-time rule (docs/accesscontrol.md §8, version-aware): a witnessed
|
|
8509
|
+
// entry is judged against the directory as it provably was at its
|
|
8510
|
+
// `receivedAt`. A versioned un-witnessed entry (provisional) has no
|
|
8511
|
+
// provable position, so it is judged against the current directory HEAD —
|
|
8512
|
+
// `MAX_SAFE_INTEGER` selects the head node deterministically regardless of
|
|
8513
|
+
// where the directory's own provisional (un-witnessed) entries currently
|
|
8514
|
+
// float, which a wall-clock `now` would not (those entries can sit at a
|
|
8515
|
+
// later `now`). A legacy un-witnessed entry predates witnessing and is
|
|
8516
|
+
// stable at its `createdAt`, so it is judged against the historical
|
|
8517
|
+
// directory state in effect then. See {@link isProvisional}.
|
|
8518
|
+
const trustedTime = (0, storeEntryTime_1.isProvisional)(meta)
|
|
8519
|
+
? Number.MAX_SAFE_INTEGER
|
|
8520
|
+
: (meta.receivedAt ?? meta.createdAt);
|
|
8521
|
+
let decision;
|
|
8522
|
+
try {
|
|
8523
|
+
decision = await evaluateClientAccess({
|
|
8524
|
+
op: meta.entryType,
|
|
8525
|
+
dbid,
|
|
8526
|
+
signingKey: entry.createdByPublicKey,
|
|
8527
|
+
trustedTime,
|
|
8528
|
+
isAuthor,
|
|
8529
|
+
beforeDoc: beforeJS,
|
|
8530
|
+
afterDoc: afterJS,
|
|
8531
|
+
});
|
|
8532
|
+
}
|
|
8533
|
+
catch (error) {
|
|
8534
|
+
// Fail closed (audit finding #2): a transient directory error must not
|
|
8535
|
+
// silently materialize an entry the policy might reject. Quarantine it
|
|
8536
|
+
// for this pass and mark the result non-cacheable so the next load
|
|
8537
|
+
// retries and self-heals once the directory is reachable again.
|
|
8538
|
+
this.logger.warn(`[ACL] evaluation failed for ${meta.id}, quarantining (will retry): ${error}`);
|
|
8539
|
+
cacheable = false;
|
|
8540
|
+
quarantine(meta, "directory_unavailable", `directory unavailable during evaluation: ${error}`);
|
|
8541
|
+
continue;
|
|
8542
|
+
}
|
|
8543
|
+
if (!decision.allowed) {
|
|
8544
|
+
quarantine(meta, decision.tier === "tier2" ? "tier2_denied" : "tier1_recheck_denied", decision.reason);
|
|
8545
|
+
continue;
|
|
8546
|
+
}
|
|
8547
|
+
// Accept the entry.
|
|
8548
|
+
currentDoc = afterDoc;
|
|
8549
|
+
if (isGenesis) {
|
|
8550
|
+
createdAt = meta.createdAt;
|
|
8551
|
+
decryptionKeyId = meta.decryptionKeyId;
|
|
8552
|
+
}
|
|
8553
|
+
acceptedMeta.push(meta);
|
|
8554
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(entry.id);
|
|
8555
|
+
if (parsed) {
|
|
8556
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, entry.id);
|
|
8557
|
+
}
|
|
8558
|
+
}
|
|
8559
|
+
if (currentDoc === null) {
|
|
8560
|
+
// The create was quarantined or absent: the document is logically absent.
|
|
8561
|
+
return { doc: null, cacheable };
|
|
8562
|
+
}
|
|
8563
|
+
const payload = currentDoc;
|
|
8564
|
+
const isDeleted = this.computeIsDeletedFromMetadata(acceptedMeta);
|
|
8565
|
+
const lastAccepted = acceptedMeta[acceptedMeta.length - 1];
|
|
8566
|
+
const lastModified = payload._lastModified || (lastAccepted ? lastAccepted.createdAt : Date.now());
|
|
8567
|
+
const witnessState = (0, storeEntryTime_1.metadataWitnessState)(acceptedMeta);
|
|
8568
|
+
return {
|
|
8569
|
+
doc: {
|
|
8570
|
+
id: docId,
|
|
8571
|
+
doc: currentDoc,
|
|
8572
|
+
createdAt: createdAt ?? lastModified,
|
|
8573
|
+
lastModified,
|
|
8574
|
+
decryptionKeyId,
|
|
8575
|
+
isDeleted,
|
|
8576
|
+
awaitingWitness: witnessState.awaitingWitness,
|
|
8577
|
+
witnessed: witnessState.witnessed,
|
|
8578
|
+
},
|
|
8579
|
+
cacheable,
|
|
8580
|
+
};
|
|
8581
|
+
}
|
|
6089
8582
|
/**
|
|
6090
8583
|
* Internal method to load a document from the content-addressed store
|
|
6091
8584
|
*/
|
|
@@ -6097,7 +8590,7 @@ class BaseMindooDB {
|
|
|
6097
8590
|
// of whether L1/L2 still hold a cached copy. This keeps the public
|
|
6098
8591
|
// contract of `loadDocumentInternal` aligned with `getDocument` /
|
|
6099
8592
|
// `getAllDocumentIds`.
|
|
6100
|
-
const indexEntryIdx = this.
|
|
8593
|
+
const indexEntryIdx = this.getDocIndexPosition(docId);
|
|
6101
8594
|
if (indexEntryIdx !== undefined && this.index[indexEntryIdx].accessState === "inaccessible") {
|
|
6102
8595
|
return null;
|
|
6103
8596
|
}
|
|
@@ -6184,6 +8677,36 @@ class BaseMindooDB {
|
|
|
6184
8677
|
// Log all entry types
|
|
6185
8678
|
const entryTypes = allEntryMetadata.map(em => `${em.entryType}@${em.createdAt}`).join(', ');
|
|
6186
8679
|
this.logger.debug(`Entry types for ${docId}: ${entryTypes}`);
|
|
8680
|
+
// Access-control materialization (docs/accesscontrol.md §10): when the
|
|
8681
|
+
// tenant has access control active, materialize through the Tier 2
|
|
8682
|
+
// quarantine path (deterministic per-entry evaluation + cascade) instead of
|
|
8683
|
+
// the optimized batch path. Returns `undefined` when ACL is not enforced,
|
|
8684
|
+
// in which case we fall through to the normal fast path below.
|
|
8685
|
+
const aclResult = await this.maybeMaterializeWithAccessControl(docId, allEntryMetadata);
|
|
8686
|
+
if (aclResult !== undefined) {
|
|
8687
|
+
const aclDoc = aclResult.doc;
|
|
8688
|
+
// Only cache a fully-validated result. A transient validation failure
|
|
8689
|
+
// (audit finding #2) leaves `cacheable` false so the next load retries.
|
|
8690
|
+
if (aclDoc && aclResult.cacheable) {
|
|
8691
|
+
await this.storeCachedDocument(aclDoc);
|
|
8692
|
+
this.markDocDirty(docId);
|
|
8693
|
+
}
|
|
8694
|
+
this.performanceCallback?.onDocumentLoad?.({
|
|
8695
|
+
docId,
|
|
8696
|
+
cacheHit: false,
|
|
8697
|
+
metadataEntriesScanned: allEntryMetadata.length,
|
|
8698
|
+
replayEntriesLoaded: allEntryMetadata.length,
|
|
8699
|
+
snapshotUsed: false,
|
|
8700
|
+
cacheCheckTime,
|
|
8701
|
+
storeQueryTime,
|
|
8702
|
+
entryLoadTime,
|
|
8703
|
+
signatureVerificationTime,
|
|
8704
|
+
decryptionTime,
|
|
8705
|
+
automergeTime,
|
|
8706
|
+
totalTime: Date.now() - startedAt,
|
|
8707
|
+
});
|
|
8708
|
+
return aclDoc;
|
|
8709
|
+
}
|
|
6187
8710
|
const metadataById = new Map(allEntryMetadata.map((meta) => [meta.id, meta]));
|
|
6188
8711
|
const planStartedAt = Date.now();
|
|
6189
8712
|
const materializationPlan = (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, allEntryMetadata, {
|
|
@@ -6238,7 +8761,7 @@ class BaseMindooDB {
|
|
|
6238
8761
|
// Verify signature against the encrypted snapshot (no decryption needed)
|
|
6239
8762
|
// We sign the encrypted payload, so anyone can verify signatures without decryption keys
|
|
6240
8763
|
const signatureStartedAt = Date.now();
|
|
6241
|
-
isValid = await this.tenant.
|
|
8764
|
+
isValid = await this.tenant.verifyEntrySignature(snapshotData, snapshotData.encryptedData);
|
|
6242
8765
|
signatureVerificationTime += Date.now() - signatureStartedAt;
|
|
6243
8766
|
}
|
|
6244
8767
|
if (!isValid) {
|
|
@@ -6260,13 +8783,25 @@ class BaseMindooDB {
|
|
|
6260
8783
|
const automergeStartedAt = Date.now();
|
|
6261
8784
|
doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
|
|
6262
8785
|
automergeTime += Date.now() - automergeStartedAt;
|
|
6263
|
-
|
|
6264
|
-
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
8786
|
+
if (this.isDebugEnabled())
|
|
8787
|
+
this.logger.debug(`Successfully loaded snapshot, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
|
|
8788
|
+
// Mandatory snapshot-head verification (docs/accesscontrol.md §10): the
|
|
8789
|
+
// decoded heads must exactly equal the declared snapshotHeadHashes, so
|
|
8790
|
+
// a snapshot cannot smuggle unauthorized content. On mismatch, discard
|
|
8791
|
+
// the snapshot and load from scratch via individually-verified entries.
|
|
8792
|
+
if (!(0, materializationGuard_1.snapshotHeadsMatch)(automerge_adapter_1.Automerge.getHeads(doc), snapshotData.snapshotHeadHashes)) {
|
|
8793
|
+
this.logger.warn(`Snapshot ${snapshotData.id} heads do not match declared snapshotHeadHashes, loading from scratch`);
|
|
8794
|
+
doc = undefined;
|
|
8795
|
+
startFromSnapshot = false;
|
|
8796
|
+
}
|
|
8797
|
+
else {
|
|
8798
|
+
// Register the snapshot's automerge hash -> entry ID mapping
|
|
8799
|
+
const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
|
|
8800
|
+
if (parsed) {
|
|
8801
|
+
this.registerAutomergeHashMapping(docId, parsed.automergeHash, snapshotData.id);
|
|
8802
|
+
}
|
|
8803
|
+
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
6268
8804
|
}
|
|
6269
|
-
this.registerSnapshotHeadHashMappings(snapshotData);
|
|
6270
8805
|
}
|
|
6271
8806
|
}
|
|
6272
8807
|
}
|
|
@@ -6274,7 +8809,8 @@ class BaseMindooDB {
|
|
|
6274
8809
|
if (!doc) {
|
|
6275
8810
|
this.logger.debug(`Initializing new Automerge document for ${docId}`);
|
|
6276
8811
|
doc = automerge_adapter_1.Automerge.init();
|
|
6277
|
-
|
|
8812
|
+
if (this.isDebugEnabled())
|
|
8813
|
+
this.logger.debug(`Initialized empty document, heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
|
|
6278
8814
|
}
|
|
6279
8815
|
// Load and apply all entries
|
|
6280
8816
|
this.logger.debug(`Fetching ${entriesToLoad.length} entries from store for document ${docId}`);
|
|
@@ -6284,7 +8820,8 @@ class BaseMindooDB {
|
|
|
6284
8820
|
this.logger.debug(`Retrieved ${entries.length} entries from store for document ${docId}`);
|
|
6285
8821
|
this.logger.debug(`Loading document ${docId}: found ${entries.length} entries to apply (${startFromSnapshot ? 'starting from snapshot' : 'starting from scratch'})`);
|
|
6286
8822
|
// Log current document state before applying entries
|
|
6287
|
-
|
|
8823
|
+
if (this.isDebugEnabled())
|
|
8824
|
+
this.logger.debug(`Document state before applying entries: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
|
|
6288
8825
|
// Filter entries for admin-only mode first
|
|
6289
8826
|
const validEntries = entries.filter(entryData => {
|
|
6290
8827
|
if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
|
|
@@ -6327,7 +8864,7 @@ class BaseMindooDB {
|
|
|
6327
8864
|
// Key was not trusted or failed to import
|
|
6328
8865
|
return { entryData, isValid: false };
|
|
6329
8866
|
}
|
|
6330
|
-
const isValid = await this.
|
|
8867
|
+
const isValid = await this.verifyEntrySignatureWithKey(cryptoKey, entryData);
|
|
6331
8868
|
return { entryData, isValid };
|
|
6332
8869
|
}));
|
|
6333
8870
|
signatureVerificationTime += Date.now() - signatureStartedAt;
|
|
@@ -6372,7 +8909,8 @@ class BaseMindooDB {
|
|
|
6372
8909
|
doc = result[0];
|
|
6373
8910
|
automergeTime += Date.now() - automergeStartedAt;
|
|
6374
8911
|
this.logger.debug(`Successfully applied ${changeBytes.length} changes to document ${docId}`);
|
|
6375
|
-
|
|
8912
|
+
if (this.isDebugEnabled())
|
|
8913
|
+
this.logger.debug(`Document state after applying changes: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
|
|
6376
8914
|
}
|
|
6377
8915
|
catch (error) {
|
|
6378
8916
|
this.logger.error(`Error applying changes to document ${docId}:`, error);
|
|
@@ -6384,7 +8922,8 @@ class BaseMindooDB {
|
|
|
6384
8922
|
}
|
|
6385
8923
|
// Extract metadata from document (doc is guaranteed to be defined at this point)
|
|
6386
8924
|
this.logger.debug(`All entries applied successfully for document ${docId}`);
|
|
6387
|
-
|
|
8925
|
+
if (this.isDebugEnabled())
|
|
8926
|
+
this.logger.debug(`Final document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
|
|
6388
8927
|
const payload = doc;
|
|
6389
8928
|
const isDeleted = this.computeIsDeletedFromMetadata(allEntryMetadata);
|
|
6390
8929
|
this.logger.debug(`Document ${docId} isDeleted: ${isDeleted}`);
|
|
@@ -6400,6 +8939,7 @@ class BaseMindooDB {
|
|
|
6400
8939
|
const firstEntry = allEntryMetadata.length > 0 ? allEntryMetadata[0] : null;
|
|
6401
8940
|
const createdAt = firstEntry ? firstEntry.createdAt : lastModified;
|
|
6402
8941
|
this.logger.debug(`Document ${docId} metadata: createdAt=${createdAt}, lastModified=${lastModified}, decryptionKeyId=${decryptionKeyId}`);
|
|
8942
|
+
const witnessState = (0, storeEntryTime_1.metadataWitnessState)(allEntryMetadata);
|
|
6403
8943
|
const internalDoc = {
|
|
6404
8944
|
id: docId,
|
|
6405
8945
|
doc: doc, // doc is guaranteed to be defined at this point
|
|
@@ -6407,6 +8947,8 @@ class BaseMindooDB {
|
|
|
6407
8947
|
lastModified,
|
|
6408
8948
|
decryptionKeyId,
|
|
6409
8949
|
isDeleted,
|
|
8950
|
+
awaitingWitness: witnessState.awaitingWitness,
|
|
8951
|
+
witnessed: witnessState.witnessed,
|
|
6410
8952
|
};
|
|
6411
8953
|
// Update cache
|
|
6412
8954
|
await this.storeCachedDocument(internalDoc);
|
|
@@ -6505,11 +9047,21 @@ class BaseMindooDB {
|
|
|
6505
9047
|
getLastModified: () => internalDoc.lastModified,
|
|
6506
9048
|
getDecryptionKeyId: () => internalDoc.decryptionKeyId,
|
|
6507
9049
|
isDeleted: () => internalDoc.isDeleted,
|
|
9050
|
+
isAccessible: () => internalDoc.accessible ?? true,
|
|
9051
|
+
isAwaitingWitness: () => internalDoc.awaitingWitness ?? false,
|
|
9052
|
+
isWitnessed: () => internalDoc.witnessed ?? false,
|
|
6508
9053
|
getHeads: () => automerge_adapter_1.Automerge.getHeads(internalDoc.doc),
|
|
6509
9054
|
getData: () => {
|
|
6510
|
-
// Convert Automerge document to plain JS object, converting Text
|
|
6511
|
-
|
|
6512
|
-
|
|
9055
|
+
// Convert Automerge document to plain JS object, converting Text
|
|
9056
|
+
// objects to strings. Memoized per InternalDoc and invalidated via
|
|
9057
|
+
// the document's Automerge heads, so repeated getData() calls on an
|
|
9058
|
+
// unchanged doc skip the full re-materialization.
|
|
9059
|
+
const headsKey = automerge_adapter_1.Automerge.getHeads(internalDoc.doc).join(",");
|
|
9060
|
+
if (internalDoc.jsCache === undefined || internalDoc.jsCacheHeads !== headsKey) {
|
|
9061
|
+
internalDoc.jsCache = this.convertAutomergeToJS(internalDoc.doc);
|
|
9062
|
+
internalDoc.jsCacheHeads = headsKey;
|
|
9063
|
+
}
|
|
9064
|
+
return createReadOnlyProxy(internalDoc.jsCache);
|
|
6513
9065
|
},
|
|
6514
9066
|
// ========== Attachment Write Methods ==========
|
|
6515
9067
|
// These throw errors in the read-only wrapper
|
|
@@ -6525,6 +9077,9 @@ class BaseMindooDB {
|
|
|
6525
9077
|
appendToAttachment: async () => {
|
|
6526
9078
|
throw new Error("appendToAttachment() can only be called within changeDoc() callback");
|
|
6527
9079
|
},
|
|
9080
|
+
setAttachmentExtractedText: () => {
|
|
9081
|
+
throw new Error("setAttachmentExtractedText() can only be called within changeDoc() callback");
|
|
9082
|
+
},
|
|
6528
9083
|
// ========== Attachment Read Methods ==========
|
|
6529
9084
|
// These work in the read-only wrapper
|
|
6530
9085
|
getAttachments: () => {
|
|
@@ -6544,6 +9099,50 @@ class BaseMindooDB {
|
|
|
6544
9099
|
this.wrappedInternalDocs.set(wrapped, internalDoc);
|
|
6545
9100
|
return wrapped;
|
|
6546
9101
|
}
|
|
9102
|
+
/**
|
|
9103
|
+
* Build a lightweight read-only tombstone for a now-inaccessible index entry
|
|
9104
|
+
* (docs/accesscontrol.md §13): a document that was readable before but whose
|
|
9105
|
+
* decryption key the current KeyBag no longer holds (e.g. a revoked key).
|
|
9106
|
+
* Emitted by {@link iterateChangesSince} so incremental consumers get a
|
|
9107
|
+
* removal signal. Per the design rule it reports `isDeleted() === true`,
|
|
9108
|
+
* `isAccessible() === false`, `getData() === {}` (the plaintext is purged,
|
|
9109
|
+
* never materialized) and `getHeads() === []` — backed by an empty Automerge
|
|
9110
|
+
* document so it never touches the underlying ciphertext or the document cache.
|
|
9111
|
+
*/
|
|
9112
|
+
buildInaccessibleDoc(entry) {
|
|
9113
|
+
const internalDoc = {
|
|
9114
|
+
id: entry.docId,
|
|
9115
|
+
doc: automerge_adapter_1.Automerge.init(),
|
|
9116
|
+
createdAt: 0,
|
|
9117
|
+
lastModified: entry.lastModified,
|
|
9118
|
+
decryptionKeyId: entry.decryptionKeyId,
|
|
9119
|
+
isDeleted: true,
|
|
9120
|
+
accessible: false,
|
|
9121
|
+
};
|
|
9122
|
+
return this.wrapDocument(internalDoc);
|
|
9123
|
+
}
|
|
9124
|
+
/**
|
|
9125
|
+
* Build a lightweight read-only tombstone for a deleted document, emitted
|
|
9126
|
+
* by {@link iterateChangesSince} instead of materializing the deleted
|
|
9127
|
+
* body. Reports `isDeleted() === true`, `isAccessible() === true` and
|
|
9128
|
+
* `getData() === {}` — the counterpart to {@link buildInaccessibleDoc} for
|
|
9129
|
+
* genuine deletions (distinguish the two via `isAccessible()`). Consumers
|
|
9130
|
+
* that need the last state of a deleted document can use
|
|
9131
|
+
* {@link iterateDocumentHistory} or {@link getDocumentAtTimestamp}.
|
|
9132
|
+
*/
|
|
9133
|
+
buildDeletedDoc(entry) {
|
|
9134
|
+
const internalDoc = {
|
|
9135
|
+
id: entry.docId,
|
|
9136
|
+
doc: automerge_adapter_1.Automerge.init(),
|
|
9137
|
+
createdAt: 0,
|
|
9138
|
+
lastModified: entry.lastModified,
|
|
9139
|
+
decryptionKeyId: entry.decryptionKeyId,
|
|
9140
|
+
isDeleted: true,
|
|
9141
|
+
awaitingWitness: entry.awaitingWitness ?? false,
|
|
9142
|
+
witnessed: entry.witnessed ?? false,
|
|
9143
|
+
};
|
|
9144
|
+
return this.wrapDocument(internalDoc);
|
|
9145
|
+
}
|
|
6547
9146
|
/**
|
|
6548
9147
|
* Get an attachment reference by ID from a document's _attachments array.
|
|
6549
9148
|
*/
|
|
@@ -6586,7 +9185,7 @@ class BaseMindooDB {
|
|
|
6586
9185
|
if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
|
|
6587
9186
|
throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
|
|
6588
9187
|
}
|
|
6589
|
-
const isValid = await this.tenant.
|
|
9188
|
+
const isValid = await this.tenant.verifyEntrySignature(chunk, chunk.encryptedData);
|
|
6590
9189
|
if (!isValid) {
|
|
6591
9190
|
throw new Error(`Invalid signature for chunk ${chunk.id}`);
|
|
6592
9191
|
}
|
|
@@ -6755,6 +9354,8 @@ class BaseMindooDB {
|
|
|
6755
9354
|
encryptedSize: encryptedData.length,
|
|
6756
9355
|
encryptedData,
|
|
6757
9356
|
};
|
|
9357
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
9358
|
+
chunkEntry.metadataSignature = await this.computeEntryMetadataSignature(chunkEntry);
|
|
6758
9359
|
chunks.push(chunkEntry);
|
|
6759
9360
|
prevChunkId = chunkId;
|
|
6760
9361
|
}
|
|
@@ -6799,22 +9400,25 @@ class BaseMindooDB {
|
|
|
6799
9400
|
const encryptedData = new Uint8Array(0);
|
|
6800
9401
|
const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedData, this.getSubtle());
|
|
6801
9402
|
const signature = await this.tenant.signPayload(encryptedData);
|
|
6802
|
-
|
|
6803
|
-
|
|
6804
|
-
|
|
6805
|
-
|
|
6806
|
-
|
|
6807
|
-
|
|
6808
|
-
|
|
6809
|
-
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
|
|
6815
|
-
|
|
6816
|
-
|
|
6817
|
-
|
|
9403
|
+
const ledgerEntry = {
|
|
9404
|
+
entryType: "pending_attachment_upload",
|
|
9405
|
+
id: this.pendingAttachmentUploadLedgerId(attachmentId),
|
|
9406
|
+
contentHash,
|
|
9407
|
+
docId,
|
|
9408
|
+
dependencyIds: [],
|
|
9409
|
+
createdAt,
|
|
9410
|
+
attachmentId,
|
|
9411
|
+
uploadStartedAt: Date.now(),
|
|
9412
|
+
createdByPublicKey,
|
|
9413
|
+
decryptionKeyId,
|
|
9414
|
+
signature,
|
|
9415
|
+
originalSize: 0,
|
|
9416
|
+
encryptedSize: 0,
|
|
9417
|
+
encryptedData,
|
|
9418
|
+
};
|
|
9419
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
9420
|
+
ledgerEntry.metadataSignature = await this.computeEntryMetadataSignature(ledgerEntry);
|
|
9421
|
+
await this.putEntriesWithRetry(this.store, [ledgerEntry], `pending ledger ${attachmentId}`);
|
|
6818
9422
|
}
|
|
6819
9423
|
async clearPendingAttachmentUploadLedger(attachmentId) {
|
|
6820
9424
|
const deleteEntriesById = this.store.deleteEntriesById;
|
|
@@ -6887,6 +9491,8 @@ class BaseMindooDB {
|
|
|
6887
9491
|
encryptedSize: encryptedData.length,
|
|
6888
9492
|
encryptedData,
|
|
6889
9493
|
};
|
|
9494
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
9495
|
+
chunkEntry.metadataSignature = await this.computeEntryMetadataSignature(chunkEntry);
|
|
6890
9496
|
pendingEntries.push(chunkEntry);
|
|
6891
9497
|
pendingEncryptedBytes += encryptedData.length;
|
|
6892
9498
|
chunkCount++;
|
|
@@ -6981,6 +9587,8 @@ class BaseMindooDB {
|
|
|
6981
9587
|
encryptedSize: encryptedData.length,
|
|
6982
9588
|
encryptedData,
|
|
6983
9589
|
};
|
|
9590
|
+
// Bind the metadata with the author signature (audit finding #5).
|
|
9591
|
+
chunkEntry.metadataSignature = await this.computeEntryMetadataSignature(chunkEntry);
|
|
6984
9592
|
chunks.push(chunkEntry);
|
|
6985
9593
|
prevChunkId = chunkId;
|
|
6986
9594
|
}
|
|
@@ -7113,6 +9721,16 @@ class BaseMindooDB {
|
|
|
7113
9721
|
}
|
|
7114
9722
|
if (storeKind === types_2.StoreKind.docs) {
|
|
7115
9723
|
await this.syncStoreChanges();
|
|
9724
|
+
// SDK-driven key-distribution reconcile (docs/accesscontrol.md §13).
|
|
9725
|
+
// After the directory pulls and processes new entries, reconcile the
|
|
9726
|
+
// local KeyBag against the directory head: bulk-remove revoked keys
|
|
9727
|
+
// (forgetting now-inaccessible docs) and import keys newly pushed to
|
|
9728
|
+
// this user (revealing their docs on the next content-DB pull). Runs in
|
|
9729
|
+
// the SDK so standalone apps get it too, not just Haven. Best-effort and
|
|
9730
|
+
// single-flight; never blocks or fails the sync.
|
|
9731
|
+
if (this.store.getId() === "directory") {
|
|
9732
|
+
await this.tenant.reconcileKeyDistributionsForCurrentUserSafe();
|
|
9733
|
+
}
|
|
7116
9734
|
}
|
|
7117
9735
|
if (options?.signal?.aborted) {
|
|
7118
9736
|
this.logger.info(`Pull cancelled after local processing for ${storeKind} entries`);
|
|
@@ -7199,4 +9817,11 @@ class BaseMindooDB {
|
|
|
7199
9817
|
}
|
|
7200
9818
|
}
|
|
7201
9819
|
exports.BaseMindooDB = BaseMindooDB;
|
|
9820
|
+
/** The "access control not enforced" allow decision returned by the
|
|
9821
|
+
* non-throwing `canCreate`/`canChange`/`canDelete` prediction helpers. */
|
|
9822
|
+
BaseMindooDB.ACL_NOT_ENFORCED_DECISION = {
|
|
9823
|
+
allowed: true,
|
|
9824
|
+
reason: "access control not enforced",
|
|
9825
|
+
tier: "tier1",
|
|
9826
|
+
};
|
|
7202
9827
|
//# sourceMappingURL=BaseMindooDB.js.map
|