@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cliOperations.ts +12 -0
- package/bin/restart.ts +66 -6
- package/components/Application.ts +1134 -112
- package/components/DESIGN.md +421 -0
- package/components/OptionsWatcher.ts +368 -102
- package/components/Scope.ts +20 -1
- package/components/componentLoader.ts +43 -4
- package/components/deploymentOperations.ts +4 -1
- package/components/deploymentRecorder.ts +9 -2
- package/components/mcp/DESIGN.md +109 -0
- package/components/mcp/audit.ts +21 -17
- package/components/operations.js +284 -52
- package/components/operationsValidation.js +49 -2
- package/components/packageComponent.ts +25 -1
- package/components/requestRestart.ts +11 -0
- package/config/DESIGN.md +306 -0
- package/config/RootConfigWatcher.ts +191 -37
- package/config/configReadRetry.ts +62 -0
- package/config/configUtils.ts +78 -26
- package/config/parseConfigFile.ts +34 -0
- package/config/readConfigFileSync.ts +44 -0
- package/config/watcherArming.ts +59 -0
- package/config-root.schema.json +4 -0
- package/dataLayer/DESIGN.md +179 -0
- package/dataLayer/harperBridge/ResourceBridge.ts +28 -2
- package/dataLayer/restoreMarker.ts +92 -25
- package/dist/bin/cliOperations.js +13 -0
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/restart.js +42 -6
- package/dist/bin/restart.js.map +1 -1
- package/dist/components/Application.d.ts +104 -9
- package/dist/components/Application.js +954 -102
- package/dist/components/Application.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +4 -1
- package/dist/components/OptionsWatcher.js +378 -104
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/Scope.js +15 -1
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/componentLoader.js +35 -3
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/deploymentOperations.js +4 -1
- package/dist/components/deploymentOperations.js.map +1 -1
- package/dist/components/deploymentRecorder.d.ts +4 -2
- package/dist/components/deploymentRecorder.js +1 -0
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/audit.d.ts +2 -1
- package/dist/components/mcp/audit.js +21 -17
- package/dist/components/mcp/audit.js.map +1 -1
- package/dist/components/operations.d.ts +28 -0
- package/dist/components/operations.js +263 -54
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +48 -2
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/packageComponent.js +24 -0
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/components/requestRestart.d.ts +1 -0
- package/dist/components/requestRestart.js +7 -0
- package/dist/components/requestRestart.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +2 -0
- package/dist/config/RootConfigWatcher.js +189 -35
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configReadRetry.d.ts +8 -0
- package/dist/config/configReadRetry.js +62 -0
- package/dist/config/configReadRetry.js.map +1 -0
- package/dist/config/configUtils.d.ts +10 -9
- package/dist/config/configUtils.js +63 -27
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +4 -0
- package/dist/config/parseConfigFile.js +35 -0
- package/dist/config/parseConfigFile.js.map +1 -0
- package/dist/config/readConfigFileSync.d.ts +1 -0
- package/dist/config/readConfigFileSync.js +47 -0
- package/dist/config/readConfigFileSync.js.map +1 -0
- package/dist/config/watcherArming.d.ts +15 -0
- package/dist/config/watcherArming.js +59 -0
- package/dist/config/watcherArming.js.map +1 -0
- package/dist/dataLayer/harperBridge/ResourceBridge.js +21 -2
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/restoreMarker.d.ts +21 -8
- package/dist/dataLayer/restoreMarker.js +94 -27
- package/dist/dataLayer/restoreMarker.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +3 -0
- package/dist/resources/DatabaseTransaction.d.ts +25 -0
- package/dist/resources/DatabaseTransaction.js +224 -18
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +2 -1
- package/dist/resources/LMDBTransaction.js +22 -3
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.js +22 -6
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +1 -1
- package/dist/resources/RecordEncoder.js +46 -10
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/Resource.js +97 -13
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +8 -0
- package/dist/resources/RocksIndexStore.js +2 -1
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +10 -0
- package/dist/resources/RocksTransactionLogStore.js +104 -33
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +58 -7
- package/dist/resources/Table.js +1143 -348
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.d.ts +3 -0
- package/dist/resources/analytics/write.js +49 -16
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +170 -0
- package/dist/resources/auditStore.js +457 -11
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/crdt.d.ts +10 -0
- package/dist/resources/crdt.js +22 -0
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/dataLoader.js +3 -4
- package/dist/resources/dataLoader.js.map +1 -1
- package/dist/resources/databases.d.ts +16 -13
- package/dist/resources/databases.js +625 -177
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/derivedIndexRegistry.d.ts +5 -0
- package/dist/resources/derivedIndexRegistry.js +68 -0
- package/dist/resources/derivedIndexRegistry.js.map +1 -0
- package/dist/resources/derivedIndexRuntime.d.ts +215 -0
- package/dist/resources/derivedIndexRuntime.js +2027 -0
- package/dist/resources/derivedIndexRuntime.js.map +1 -0
- package/dist/resources/graphql.js +3 -2
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +102 -9
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +867 -39
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/indexes/fullTextDerivedIndex.d.ts +81 -0
- package/dist/resources/indexes/fullTextDerivedIndex.js +1004 -0
- package/dist/resources/indexes/fullTextDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/fullTextNativeBinding.d.ts +78 -0
- package/dist/resources/indexes/fullTextNativeBinding.js +85 -0
- package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -0
- package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
- package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
- package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
- package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
- package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
- package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.d.ts +24 -0
- package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js +149 -0
- package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js.map +1 -0
- package/dist/resources/nodeIdMapping.d.ts +5 -0
- package/dist/resources/nodeIdMapping.js +49 -0
- package/dist/resources/nodeIdMapping.js.map +1 -1
- package/dist/resources/recordLock.d.ts +47 -4
- package/dist/resources/recordLock.js +138 -7
- package/dist/resources/recordLock.js.map +1 -1
- package/dist/resources/recordLockCoordinator.d.ts +557 -0
- package/dist/resources/recordLockCoordinator.js +2597 -0
- package/dist/resources/recordLockCoordinator.js.map +1 -0
- package/dist/resources/replayLogs.js +5 -0
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replicatedApplyFailure.d.ts +16 -0
- package/dist/resources/replicatedApplyFailure.js +63 -0
- package/dist/resources/replicatedApplyFailure.js.map +1 -0
- package/dist/resources/scheduler/scheduler.js +3 -3
- package/dist/resources/scheduler/scheduler.js.map +1 -1
- package/dist/resources/search.d.ts +10 -4
- package/dist/resources/search.js +160 -40
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/tracked.d.ts +5 -1
- package/dist/resources/tracked.js +74 -23
- package/dist/resources/tracked.js.map +1 -1
- package/dist/resources/transactionBroadcast.js +4 -6
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +59 -23
- package/dist/security/auth.js.map +1 -1
- package/dist/security/deferredAuthentication.d.ts +11 -0
- package/dist/security/deferredAuthentication.js +25 -3
- package/dist/security/deferredAuthentication.js.map +1 -1
- package/dist/security/jsLoader.js +9 -4
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/server/REST.js +39 -5
- package/dist/server/REST.js.map +1 -1
- package/dist/server/http.d.ts +5 -1
- package/dist/server/http.js +34 -2
- package/dist/server/http.js.map +1 -1
- package/dist/server/mqtt.js +5 -1
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +2 -0
- package/dist/server/serverHelpers/Headers.js +6 -0
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
- package/dist/server/serverHelpers/Request.d.ts +5 -10
- package/dist/server/serverHelpers/Request.js +38 -136
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +2 -0
- package/dist/server/serverHelpers/contentTypes.js +189 -15
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +3 -3
- package/dist/server/serverHelpers/serverUtilities.js +110 -20
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.js +4 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/serverHelpers/webSocketCloseReason.d.ts +2 -0
- package/dist/server/serverHelpers/webSocketCloseReason.js +29 -0
- package/dist/server/serverHelpers/webSocketCloseReason.js.map +1 -0
- package/dist/server/storageReclamation.js +1 -1
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/isolatedApplications.d.ts +47 -0
- package/dist/server/threads/isolatedApplications.js +171 -0
- package/dist/server/threads/isolatedApplications.js.map +1 -0
- package/dist/server/threads/logRotationTransport.d.ts +1 -0
- package/dist/server/threads/logRotationTransport.js +33 -0
- package/dist/server/threads/logRotationTransport.js.map +1 -0
- package/dist/server/threads/manageThreads.d.ts +64 -6
- package/dist/server/threads/manageThreads.js +261 -12
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -0
- package/dist/server/threads/socketRouter.js +196 -13
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadServer.js +30 -7
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +24 -0
- package/dist/utility/errors/hdbError.js +58 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/hdbTerms.d.ts +2 -0
- package/dist/utility/hdbTerms.js +2 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.js +218 -38
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
- package/dist/utility/logging/logGenerationCoordinator.js +184 -0
- package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
- package/dist/utility/logging/logRotation.d.ts +46 -0
- package/dist/utility/logging/logRotation.js +365 -0
- package/dist/utility/logging/logRotation.js.map +1 -0
- package/dist/utility/logging/logRotator.d.ts +1 -1
- package/dist/utility/logging/logRotator.js +172 -92
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/npmUtilities.js +6 -4
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/watcherFallback.d.ts +0 -45
- package/dist/utility/watcherFallback.js +1 -125
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.js +6 -3
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +9 -0
- package/json/systemSchema.json +3 -0
- package/npm-shrinkwrap.json +177 -87
- package/package.json +13 -5
- package/resources/DESIGN.md +689 -19
- package/resources/DatabaseTransaction.ts +230 -17
- package/resources/LMDBTransaction.ts +21 -3
- package/resources/PrimaryRocksDatabase.ts +20 -7
- package/resources/RecordEncoder.ts +52 -10
- package/resources/Resource.ts +97 -13
- package/resources/ResourceInterface.ts +8 -0
- package/resources/RocksIndexStore.ts +2 -1
- package/resources/RocksTransactionLogStore.ts +111 -31
- package/resources/Table.ts +1273 -396
- package/resources/analytics/DESIGN.md +38 -0
- package/resources/analytics/write.ts +50 -17
- package/resources/auditStore.ts +460 -11
- package/resources/crdt.ts +22 -0
- package/resources/dataLoader.ts +3 -4
- package/resources/databases.ts +611 -147
- package/resources/derivedIndexRegistry.ts +56 -0
- package/resources/derivedIndexRuntime.ts +2292 -0
- package/resources/graphql.ts +3 -2
- package/resources/indexes/DESIGN.md +833 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +924 -46
- package/resources/indexes/fullTextDerivedIndex.ts +1165 -0
- package/resources/indexes/fullTextNativeBinding.ts +146 -0
- package/resources/indexes/hnswDerivedIndex.ts +531 -0
- package/resources/indexes/hnswPlaneBinding.ts +174 -0
- package/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts +181 -0
- package/resources/nodeIdMapping.ts +50 -0
- package/resources/record-locks.md +1407 -0
- package/resources/recordLock.ts +173 -7
- package/resources/recordLockCoordinator.ts +3082 -0
- package/resources/replayLogs.ts +5 -0
- package/resources/replicatedApplyFailure.ts +77 -0
- package/resources/scheduler/DESIGN.md +40 -0
- package/resources/scheduler/scheduler.ts +4 -4
- package/resources/search.ts +169 -49
- package/resources/tracked.ts +73 -22
- package/resources/transactionBroadcast.ts +4 -4
- package/security/DESIGN.md +175 -0
- package/security/auth.ts +53 -24
- package/security/deferredAuthentication.ts +24 -2
- package/security/jsLoader.ts +9 -4
- package/server/DESIGN.md +275 -0
- package/server/REST.ts +42 -6
- package/server/http.ts +34 -2
- package/server/mqtt.ts +6 -4
- package/server/serverHelpers/Headers.ts +5 -1
- package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
- package/server/serverHelpers/Request.ts +33 -131
- package/server/serverHelpers/contentTypes.ts +188 -15
- package/server/serverHelpers/serverUtilities.ts +157 -27
- package/server/serverHelpers/uwsServer.ts +4 -1
- package/server/serverHelpers/webSocketCloseReason.ts +25 -0
- package/server/storageReclamation.ts +2 -2
- package/server/threads/isolatedApplications.ts +157 -0
- package/server/threads/logRotationTransport.ts +40 -0
- package/server/threads/manageThreads.js +254 -12
- package/server/threads/socketRouter.ts +217 -11
- package/server/threads/threadServer.js +30 -7
- package/studio/web/assets/{Chat-BnCBegQz.js → Chat-DADFFGe_.js} +1 -1
- package/studio/web/assets/{FloatingChat-CoDW1ySS.js → FloatingChat-D_mI-rZ7.js} +3 -3
- package/studio/web/assets/{apiToken-Bwk5BLXW.js → apiToken-c2NiSDHa.js} +1 -1
- package/studio/web/assets/{applications-DHxGi7JH.js → applications-DktUqh7G.js} +1 -1
- package/studio/web/assets/{cssMode-s0cWI-_M.js → cssMode-Cs_75Xhw.js} +1 -1
- package/studio/web/assets/{editor-DNcRHK54.js → editor-19b-Y1IN.js} +1 -1
- package/studio/web/assets/{html-Bdssedlg.js → html-DiYEQMpB.js} +1 -1
- package/studio/web/assets/{htmlMode-CoDlJ3fw.js → htmlMode-CmR0y7P_.js} +1 -1
- package/studio/web/assets/{index-D6sxmFLR.js → index-Dm0rfkJ7.js} +5 -5
- package/studio/web/assets/{index.lazy-tmU5BS8s.js → index.lazy-7vqt2CC3.js} +1 -1
- package/studio/web/assets/{javascript-B8meVSTH.js → javascript-BWtCFuOt.js} +1 -1
- package/studio/web/assets/{jsonMode-DpIPd35T.js → jsonMode-Buzzbv9y.js} +1 -1
- package/studio/web/assets/{languageServices-C_5FMJzQ.js → languageServices-SqsFWfTM.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-BIzNBzPK.js → lspLanguageFeatures-EMV5cmjo.js} +1 -1
- package/studio/web/assets/{notifications-CvZivSbh.js → notifications-CAB-LZWT.js} +1 -1
- package/studio/web/assets/{notifications-CQf18QKb.js → notifications-DRzmSRxM.js} +1 -1
- package/studio/web/assets/{profile-DdOwtntb.js → profile-BNKAl79n.js} +1 -1
- package/studio/web/assets/{regions-n69fwagr.js → regions-CUow_Zw2.js} +1 -1
- package/studio/web/assets/{register-PfWTCXWB.js → register-Dkt3WUMp.js} +2 -2
- package/studio/web/assets/{setComponentFile-Bg6O7X0S.js → setComponentFile-BZRfMD0N.js} +1 -1
- package/studio/web/assets/{setup-CUx_aUDl.js → setup-D_yiEPO2.js} +2 -2
- package/studio/web/assets/{status-D7BVKqX9.js → status-DhHh1Ge-.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-DBSyXzMr.js → toggleHighContrast-D7L1PDtV.js} +1 -1
- package/studio/web/assets/{tsMode-BByKCjBS.js → tsMode-CCwLk1YS.js} +1 -1
- package/studio/web/assets/{typescript-DDLnLpw9.js → typescript-BP1j1mjn.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-31CHGaHk.js → useEntityRestURL-D7bnYxLw.js} +1 -1
- package/studio/web/assets/{workers-pR3jRY9D.js → workers-tOuCNT17.js} +1 -1
- package/studio/web/assets/{xml-2iRnMhQO.js → xml-BSG_3mQT.js} +1 -1
- package/studio/web/assets/{yaml-Bf92gJpd.js → yaml-DxiLprBB.js} +1 -1
- package/studio/web/index.html +1 -1
- package/utility/DESIGN.md +55 -0
- package/utility/errors/hdbError.ts +54 -0
- package/utility/hdbTerms.ts +2 -0
- package/utility/logging/harper_logger.ts +210 -30
- package/utility/logging/logGenerationCoordinator.ts +196 -0
- package/utility/logging/logRotation.ts +367 -0
- package/utility/logging/logRotator.ts +196 -91
- package/utility/npmUtilities.ts +6 -4
- package/utility/watcherFallback.ts +0 -122
- package/validation/configValidator.ts +6 -3
|
@@ -0,0 +1,1165 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DERIVED_INDEX_ACCEPTED,
|
|
3
|
+
DERIVED_INDEX_DEFERRED,
|
|
4
|
+
DERIVED_INDEX_FAILED,
|
|
5
|
+
sameDerivedIndexPositions,
|
|
6
|
+
} from '../derivedIndexRuntime.ts';
|
|
7
|
+
import type {
|
|
8
|
+
DerivedIndexBackend,
|
|
9
|
+
DerivedIndexBackendHost,
|
|
10
|
+
DerivedIndexBackendStateChange,
|
|
11
|
+
DerivedIndexBatch,
|
|
12
|
+
DerivedIndexCursor,
|
|
13
|
+
DerivedIndexDeliveryResult,
|
|
14
|
+
DerivedIndexFlushReason,
|
|
15
|
+
DerivedIndexPositions,
|
|
16
|
+
} from '../derivedIndexRuntime.ts';
|
|
17
|
+
import { toBufferKey } from 'ordered-binary';
|
|
18
|
+
import { loggerWithTag } from '../../utility/logging/logger.ts';
|
|
19
|
+
|
|
20
|
+
const logger = loggerWithTag('fulltext-derived-index');
|
|
21
|
+
|
|
22
|
+
const DEFAULT_MAX_QUEUED_BATCHES = 16;
|
|
23
|
+
export const HARPER_FULLTEXT_DEFAULT_MAX_QUEUED_BYTES = 64 * 1024 * 1024;
|
|
24
|
+
export const HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES = 64 * 1024;
|
|
25
|
+
const DEFAULT_OPEN_ATTEMPTS = 3;
|
|
26
|
+
const DEFAULT_OPEN_RETRY_MILLISECONDS = 10;
|
|
27
|
+
const DEFAULT_MAX_OPEN_RETRY_MILLISECONDS = 5_000;
|
|
28
|
+
const DEFAULT_CLOSE_TIMEOUT_MILLISECONDS = 35_000;
|
|
29
|
+
const DEFAULT_MAX_APPLY_SLICE_RECORDS = 256;
|
|
30
|
+
const APPLY_SLICE_MILLISECONDS = 5;
|
|
31
|
+
const TERMINAL_OPEN_ERROR_CODES = new Set([
|
|
32
|
+
'E_IDENTITY_MISMATCH',
|
|
33
|
+
'E_INCOMPLETE_CREATE',
|
|
34
|
+
'E_INDEX_CORRUPT',
|
|
35
|
+
'E_INDEX_FORMAT_INCOMPATIBLE',
|
|
36
|
+
'E_SCHEMA_MISMATCH',
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
export interface FullTextDerivedIndexEngine {
|
|
40
|
+
readonly committedPayload?: string;
|
|
41
|
+
applyMutationBatch(
|
|
42
|
+
batch: FullTextMutationBatch,
|
|
43
|
+
options: { assumeDistinctIds: true; rejectedUpsert: 'delete' }
|
|
44
|
+
): Promise<{
|
|
45
|
+
/** Counts every logical mutation handled, including rejected upserts replaced by deletes. */
|
|
46
|
+
processed: number;
|
|
47
|
+
rejected: Array<{ operation: 'upsert'; index: number; code: 'E_INVALID_ARGUMENT' | 'E_BATCH_TOO_LARGE' }>;
|
|
48
|
+
encodedBytes: number;
|
|
49
|
+
frames: number;
|
|
50
|
+
}>;
|
|
51
|
+
/** Success proves the cursor payload and every preceding mutation are durably ordered together. */
|
|
52
|
+
publish(payload: string): Promise<bigint>;
|
|
53
|
+
close(options?: { mode?: 'require-clean' | 'rollback' }): Promise<{ cleanupError?: unknown }>;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export type FullTextDerivedIndexInspection =
|
|
57
|
+
| { state: 'missing' | 'cursorless' }
|
|
58
|
+
| { state: 'checkpointed'; committedPayload: string }
|
|
59
|
+
| { state: 'incompatible'; code: string };
|
|
60
|
+
|
|
61
|
+
export interface FullTextDerivedIndexLifecycle {
|
|
62
|
+
inspect(): FullTextDerivedIndexInspection;
|
|
63
|
+
open(): Promise<FullTextDerivedIndexEngine>;
|
|
64
|
+
reset(): Promise<void>;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export type FullTextMutationBatch = {
|
|
68
|
+
upserts: Array<{ id: string; fields: Record<string, string | string[]> }>;
|
|
69
|
+
deletes: string[];
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
export type FullTextDerivedIndexBackendOptions = {
|
|
73
|
+
id: string;
|
|
74
|
+
lifecycle: FullTextDerivedIndexLifecycle;
|
|
75
|
+
maxQueuedBatches?: number;
|
|
76
|
+
maxQueuedBytes?: number;
|
|
77
|
+
maxCursorPayloadBytes?: number;
|
|
78
|
+
maxApplySliceRecords?: number;
|
|
79
|
+
openAttempts?: number;
|
|
80
|
+
openRetryMilliseconds?: number;
|
|
81
|
+
maxOpenRetryMilliseconds?: number;
|
|
82
|
+
closeTimeoutMilliseconds?: number;
|
|
83
|
+
shutdownTimeoutMilliseconds?: number;
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
type ApplyCommand = {
|
|
87
|
+
type: 'apply';
|
|
88
|
+
epoch: bigint;
|
|
89
|
+
sequence: number;
|
|
90
|
+
batch: DerivedIndexBatch;
|
|
91
|
+
bytes: number;
|
|
92
|
+
position: number;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
type BarrierCommand = {
|
|
96
|
+
type: 'barrier';
|
|
97
|
+
epoch: bigint;
|
|
98
|
+
horizon: number;
|
|
99
|
+
cursor?: DerivedIndexCursor;
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
type Command = ApplyCommand | BarrierCommand;
|
|
103
|
+
|
|
104
|
+
type ShutdownRequest = {
|
|
105
|
+
epoch: bigint;
|
|
106
|
+
promise: Promise<void>;
|
|
107
|
+
resolve: () => void;
|
|
108
|
+
reject: (error: unknown) => void;
|
|
109
|
+
closing: boolean;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
export class FullTextDerivedIndexError extends Error {
|
|
113
|
+
statusCode = 500;
|
|
114
|
+
|
|
115
|
+
constructor(message: string) {
|
|
116
|
+
super(message);
|
|
117
|
+
this.name = 'FullTextDerivedIndexError';
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
class FullTextDerivedIndexProtocolError extends FullTextDerivedIndexError {}
|
|
122
|
+
class FullTextDerivedIndexConfigurationError extends FullTextDerivedIndexError {}
|
|
123
|
+
|
|
124
|
+
export class FullTextDerivedIndexBackend implements DerivedIndexBackend {
|
|
125
|
+
readonly id: string;
|
|
126
|
+
#lifecycle: FullTextDerivedIndexLifecycle;
|
|
127
|
+
#maxQueuedBatches: number;
|
|
128
|
+
#maxQueuedBytes: number;
|
|
129
|
+
#maxCursorPayloadBytes: number;
|
|
130
|
+
#maxApplySliceRecords: number;
|
|
131
|
+
#openAttempts: number;
|
|
132
|
+
#openRetryMilliseconds: number;
|
|
133
|
+
#maxOpenRetryMilliseconds: number;
|
|
134
|
+
#closeTimeoutMilliseconds: number;
|
|
135
|
+
#shutdownTimeoutMilliseconds: number;
|
|
136
|
+
#host?: DerivedIndexBackendHost;
|
|
137
|
+
#wake?: (change?: DerivedIndexBackendStateChange) => void;
|
|
138
|
+
#engine?: FullTextDerivedIndexEngine;
|
|
139
|
+
#durableCursor?: DerivedIndexCursor;
|
|
140
|
+
#cursorInspectedForAcquisition = false;
|
|
141
|
+
#activeEpoch?: bigint;
|
|
142
|
+
#commands: Command[] = [];
|
|
143
|
+
#retainedBatches = 0;
|
|
144
|
+
#retainedBytes = 0;
|
|
145
|
+
#lastAcceptedSequence = 0;
|
|
146
|
+
#lastAppliedSequence = 0;
|
|
147
|
+
#lastPublishedSequence = 0;
|
|
148
|
+
#lastBarrierHorizon = 0;
|
|
149
|
+
#lastAcceptedCursor?: DerivedIndexCursor;
|
|
150
|
+
#hasStagedMutations = false;
|
|
151
|
+
#draining = false;
|
|
152
|
+
#scheduled = false;
|
|
153
|
+
#settlingWriter = false;
|
|
154
|
+
#lossPendingEpoch?: bigint;
|
|
155
|
+
#failed = false;
|
|
156
|
+
#capacityDeferred = false;
|
|
157
|
+
#shutdown?: ShutdownRequest;
|
|
158
|
+
#unindexableRecords = 0;
|
|
159
|
+
#stagedUnindexableRecords = 0;
|
|
160
|
+
#invalidEstimateWarned = false;
|
|
161
|
+
#openRetryTimer?: NodeJS.Timeout;
|
|
162
|
+
#writerRetryTimer?: NodeJS.Timeout;
|
|
163
|
+
#closeOperations = new WeakMap<FullTextDerivedIndexEngine, Promise<{ cleanupError?: unknown }>>();
|
|
164
|
+
#shutdownFailure?: FullTextDerivedIndexError;
|
|
165
|
+
#openRetryDelayMilliseconds: number;
|
|
166
|
+
#openRetryWarned = false;
|
|
167
|
+
#writerRetryDelayMilliseconds: number;
|
|
168
|
+
#writerRetryWarned = false;
|
|
169
|
+
#terminalFailure?: FullTextDerivedIndexConfigurationError;
|
|
170
|
+
#resetOperation?: Promise<void>;
|
|
171
|
+
#resetEpoch?: bigint;
|
|
172
|
+
#failedDeliveryObserved = false;
|
|
173
|
+
|
|
174
|
+
constructor(options: FullTextDerivedIndexBackendOptions) {
|
|
175
|
+
if (!options.id) throw new TypeError('Full-text derived index id is required');
|
|
176
|
+
if (
|
|
177
|
+
!options.lifecycle ||
|
|
178
|
+
typeof options.lifecycle.inspect !== 'function' ||
|
|
179
|
+
typeof options.lifecycle.open !== 'function' ||
|
|
180
|
+
typeof options.lifecycle.reset !== 'function'
|
|
181
|
+
)
|
|
182
|
+
throw new TypeError('Full-text derived index lifecycle must implement inspect(), open(), and reset()');
|
|
183
|
+
this.id = options.id;
|
|
184
|
+
this.#lifecycle = options.lifecycle;
|
|
185
|
+
this.#maxQueuedBatches = positiveInteger(
|
|
186
|
+
options.maxQueuedBatches ?? DEFAULT_MAX_QUEUED_BATCHES,
|
|
187
|
+
'maxQueuedBatches'
|
|
188
|
+
);
|
|
189
|
+
this.#maxQueuedBytes = positiveInteger(
|
|
190
|
+
options.maxQueuedBytes ?? HARPER_FULLTEXT_DEFAULT_MAX_QUEUED_BYTES,
|
|
191
|
+
'maxQueuedBytes'
|
|
192
|
+
);
|
|
193
|
+
const maxCursorPayloadBytes = positiveInteger(
|
|
194
|
+
options.maxCursorPayloadBytes ?? HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES,
|
|
195
|
+
'maxCursorPayloadBytes'
|
|
196
|
+
);
|
|
197
|
+
if (maxCursorPayloadBytes > HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES)
|
|
198
|
+
throw new RangeError(
|
|
199
|
+
`Full-text maxCursorPayloadBytes must not exceed ${HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES}`
|
|
200
|
+
);
|
|
201
|
+
this.#maxCursorPayloadBytes = maxCursorPayloadBytes;
|
|
202
|
+
this.#maxApplySliceRecords = positiveInteger(
|
|
203
|
+
options.maxApplySliceRecords ?? DEFAULT_MAX_APPLY_SLICE_RECORDS,
|
|
204
|
+
'maxApplySliceRecords'
|
|
205
|
+
);
|
|
206
|
+
this.#openAttempts = positiveInteger(options.openAttempts ?? DEFAULT_OPEN_ATTEMPTS, 'openAttempts');
|
|
207
|
+
this.#openRetryMilliseconds = nonNegativeInteger(
|
|
208
|
+
options.openRetryMilliseconds ?? DEFAULT_OPEN_RETRY_MILLISECONDS,
|
|
209
|
+
'openRetryMilliseconds'
|
|
210
|
+
);
|
|
211
|
+
this.#maxOpenRetryMilliseconds = Math.max(
|
|
212
|
+
Math.max(1, this.#openRetryMilliseconds),
|
|
213
|
+
positiveInteger(
|
|
214
|
+
options.maxOpenRetryMilliseconds ?? DEFAULT_MAX_OPEN_RETRY_MILLISECONDS,
|
|
215
|
+
'maxOpenRetryMilliseconds'
|
|
216
|
+
)
|
|
217
|
+
);
|
|
218
|
+
this.#closeTimeoutMilliseconds = positiveInteger(
|
|
219
|
+
options.closeTimeoutMilliseconds ?? DEFAULT_CLOSE_TIMEOUT_MILLISECONDS,
|
|
220
|
+
'closeTimeoutMilliseconds'
|
|
221
|
+
);
|
|
222
|
+
this.#shutdownTimeoutMilliseconds = positiveInteger(
|
|
223
|
+
options.shutdownTimeoutMilliseconds ?? this.#closeTimeoutMilliseconds * 2,
|
|
224
|
+
'shutdownTimeoutMilliseconds'
|
|
225
|
+
);
|
|
226
|
+
if (this.#shutdownTimeoutMilliseconds < this.#closeTimeoutMilliseconds * 2)
|
|
227
|
+
throw new RangeError('shutdownTimeoutMilliseconds must be at least twice closeTimeoutMilliseconds');
|
|
228
|
+
this.#openRetryDelayMilliseconds = Math.max(1, this.#openRetryMilliseconds);
|
|
229
|
+
this.#writerRetryDelayMilliseconds = Math.min(
|
|
230
|
+
this.#maxOpenRetryMilliseconds,
|
|
231
|
+
Math.max(DEFAULT_OPEN_RETRY_MILLISECONDS, this.#openRetryMilliseconds)
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
attach(host: DerivedIndexBackendHost): void {
|
|
236
|
+
if (this.#host && this.#host !== host && (this.#engine || this.#draining || this.#wake))
|
|
237
|
+
throw new Error('Full-text derived index backend is already attached');
|
|
238
|
+
this.#host = host;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
getDurableCursor(): DerivedIndexCursor | undefined {
|
|
242
|
+
this.#assertAttached();
|
|
243
|
+
if (!this.#cursorInspectedForAcquisition) {
|
|
244
|
+
this.#durableCursor = this.#inspectDurableCursor();
|
|
245
|
+
this.#cursorInspectedForAcquisition = true;
|
|
246
|
+
}
|
|
247
|
+
return this.#durableCursor;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
#inspectDurableCursor(): DerivedIndexCursor | undefined {
|
|
251
|
+
let inspection: FullTextDerivedIndexInspection;
|
|
252
|
+
try {
|
|
253
|
+
inspection = this.#lifecycle.inspect();
|
|
254
|
+
} catch (error) {
|
|
255
|
+
logNativeError('Full-text derived index state could not be inspected', error);
|
|
256
|
+
throw new FullTextDerivedIndexError('Full-text derived index state could not be inspected');
|
|
257
|
+
}
|
|
258
|
+
if (inspection.state !== 'checkpointed') {
|
|
259
|
+
if (inspection.state === 'incompatible')
|
|
260
|
+
logWarning(`Full-text derived index '${this.id}' is incompatible (${inspection.code}); rebuilding`);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
try {
|
|
264
|
+
return decodeFullTextCursorPayload(inspection.committedPayload);
|
|
265
|
+
} catch {
|
|
266
|
+
logWarning(`Full-text derived index '${this.id}' has an invalid committed cursor; rebuilding`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
getUnindexableRecords(): number {
|
|
271
|
+
return this.#unindexableRecords;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
deliver(batch: DerivedIndexBatch): DerivedIndexDeliveryResult {
|
|
275
|
+
if (this.#failed) {
|
|
276
|
+
if (this.#host?.isOwnerEpoch(batch.ownerEpoch)) this.#failedDeliveryObserved = true;
|
|
277
|
+
return DERIVED_INDEX_FAILED;
|
|
278
|
+
}
|
|
279
|
+
if (!this.#host?.isOwnerEpoch(batch.ownerEpoch)) return DERIVED_INDEX_FAILED;
|
|
280
|
+
if (this.#terminalFailure) return DERIVED_INDEX_DEFERRED;
|
|
281
|
+
if (this.#settlingWriter || this.#writerRetryTimer || this.#lossPendingEpoch === batch.ownerEpoch)
|
|
282
|
+
return DERIVED_INDEX_DEFERRED;
|
|
283
|
+
if (this.#shutdown) {
|
|
284
|
+
if (this.#shutdown.epoch === batch.ownerEpoch || this.#activeEpoch !== undefined) return DERIVED_INDEX_FAILED;
|
|
285
|
+
this.#shutdown = undefined;
|
|
286
|
+
}
|
|
287
|
+
if (this.#activeEpoch === undefined) this.#activeEpoch = batch.ownerEpoch;
|
|
288
|
+
if (this.#activeEpoch !== batch.ownerEpoch) return DERIVED_INDEX_FAILED;
|
|
289
|
+
let through: DerivedIndexCursor | undefined;
|
|
290
|
+
try {
|
|
291
|
+
through = batch.through && normalizedCursor(batch.through);
|
|
292
|
+
} catch (error) {
|
|
293
|
+
this.#markFailed(error);
|
|
294
|
+
return DERIVED_INDEX_FAILED;
|
|
295
|
+
}
|
|
296
|
+
let bytes = batch.bytes;
|
|
297
|
+
if (bytes === 0) bytes = batch.records.length === 0 ? 1 : this.#maxQueuedBytes;
|
|
298
|
+
else if (!Number.isSafeInteger(bytes) || bytes < 0) {
|
|
299
|
+
bytes = this.#maxQueuedBytes;
|
|
300
|
+
if (!this.#invalidEstimateWarned) {
|
|
301
|
+
this.#invalidEstimateWarned = true;
|
|
302
|
+
logWarning(`Full-text derived index '${this.id}' received an invalid batch byte estimate`, undefined);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
if (
|
|
306
|
+
this.#retainedBatches >= this.#maxQueuedBatches ||
|
|
307
|
+
(this.#retainedBatches > 0 && this.#retainedBytes + bytes > this.#maxQueuedBytes)
|
|
308
|
+
) {
|
|
309
|
+
this.#capacityDeferred = true;
|
|
310
|
+
return DERIVED_INDEX_DEFERRED;
|
|
311
|
+
}
|
|
312
|
+
const sequence = ++this.#lastAcceptedSequence;
|
|
313
|
+
if (through) this.#lastAcceptedCursor = through;
|
|
314
|
+
this.#commands.push({ type: 'apply', epoch: batch.ownerEpoch, sequence, batch, bytes, position: 0 });
|
|
315
|
+
this.#retainedBatches++;
|
|
316
|
+
this.#retainedBytes += bytes;
|
|
317
|
+
this.#scheduleDrain();
|
|
318
|
+
return DERIVED_INDEX_ACCEPTED;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
flush(_reason: DerivedIndexFlushReason = 'threshold'): void {
|
|
322
|
+
if (this.#activeEpoch === undefined || this.#failed || this.#shutdown) return;
|
|
323
|
+
this.#queueBarrier(this.#activeEpoch);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
shutdown(ownerEpoch: bigint): Promise<void> {
|
|
327
|
+
if (this.#shutdown?.epoch === ownerEpoch) {
|
|
328
|
+
this.#cursorInspectedForAcquisition = false;
|
|
329
|
+
return this.#boundedShutdown(this.#shutdown);
|
|
330
|
+
}
|
|
331
|
+
if (this.#resetOperation) {
|
|
332
|
+
if (this.#resetEpoch !== ownerEpoch)
|
|
333
|
+
return Promise.reject(new FullTextDerivedIndexError('Full-text reset belongs to another owner epoch'));
|
|
334
|
+
this.#cursorInspectedForAcquisition = false;
|
|
335
|
+
return withTimeout(
|
|
336
|
+
this.#resetOperation.then(() => this.shutdown(ownerEpoch)),
|
|
337
|
+
this.#shutdownTimeoutMilliseconds,
|
|
338
|
+
() => new FullTextDerivedIndexError('Full-text reset did not prove quiescence before shutdown timeout')
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
if (this.#activeEpoch !== ownerEpoch) {
|
|
342
|
+
if (this.#host?.isOwnerEpoch(ownerEpoch)) this.#cursorInspectedForAcquisition = false;
|
|
343
|
+
return Promise.resolve();
|
|
344
|
+
}
|
|
345
|
+
this.#cursorInspectedForAcquisition = false;
|
|
346
|
+
this.#clearWriterRetry();
|
|
347
|
+
this.#queueBarrier(ownerEpoch);
|
|
348
|
+
let resolve: () => void;
|
|
349
|
+
let reject: (error: unknown) => void;
|
|
350
|
+
const promise = new Promise<void>((resolvePromise, rejectPromise) => {
|
|
351
|
+
resolve = resolvePromise;
|
|
352
|
+
reject = rejectPromise;
|
|
353
|
+
});
|
|
354
|
+
this.#shutdown = { epoch: ownerEpoch, promise, resolve: resolve!, reject: reject!, closing: false };
|
|
355
|
+
this.#scheduleDrain();
|
|
356
|
+
return this.#boundedShutdown(this.#shutdown);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
#boundedShutdown(request: ShutdownRequest): Promise<void> {
|
|
360
|
+
return withTimeout(
|
|
361
|
+
request.promise,
|
|
362
|
+
this.#shutdownTimeoutMilliseconds,
|
|
363
|
+
() => new FullTextDerivedIndexError('Full-text writer shutdown did not prove quiescence before timeout')
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async reset(ownerEpoch: bigint): Promise<void> {
|
|
368
|
+
this.#assertAttached();
|
|
369
|
+
if (this.#resetOperation) {
|
|
370
|
+
this.#assertSharedEpoch(ownerEpoch);
|
|
371
|
+
if (this.#resetEpoch !== ownerEpoch)
|
|
372
|
+
throw new FullTextDerivedIndexError('Full-text reset belongs to another owner epoch');
|
|
373
|
+
return this.#boundedReset(this.#resetOperation);
|
|
374
|
+
}
|
|
375
|
+
if (
|
|
376
|
+
this.#engine ||
|
|
377
|
+
this.#scheduled ||
|
|
378
|
+
this.#draining ||
|
|
379
|
+
this.#settlingWriter ||
|
|
380
|
+
this.#openRetryTimer ||
|
|
381
|
+
this.#writerRetryTimer ||
|
|
382
|
+
this.#commands.length > 0 ||
|
|
383
|
+
this.#shutdown?.closing
|
|
384
|
+
)
|
|
385
|
+
throw new FullTextDerivedIndexError('Full-text derived index backend is not quiescent at reset');
|
|
386
|
+
this.#assertSharedEpoch(ownerEpoch);
|
|
387
|
+
if (this.#terminalFailure)
|
|
388
|
+
throw new FullTextDerivedIndexError(
|
|
389
|
+
'Full-text derived index configuration must change before rebuild can succeed'
|
|
390
|
+
);
|
|
391
|
+
this.#shutdown = undefined;
|
|
392
|
+
this.#failed = false;
|
|
393
|
+
this.#shutdownFailure = undefined;
|
|
394
|
+
this.#durableCursor = undefined;
|
|
395
|
+
// An unknown reset outcome must never rediscover and trust the pre-reset checkpoint.
|
|
396
|
+
this.#cursorInspectedForAcquisition = true;
|
|
397
|
+
const operation = this.#performReset(ownerEpoch);
|
|
398
|
+
this.#resetOperation = operation;
|
|
399
|
+
this.#resetEpoch = ownerEpoch;
|
|
400
|
+
void operation.then(
|
|
401
|
+
() => this.#clearResetOperation(operation),
|
|
402
|
+
() => this.#clearResetOperation(operation)
|
|
403
|
+
);
|
|
404
|
+
return this.#boundedReset(operation);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
async #performReset(ownerEpoch: bigint): Promise<void> {
|
|
408
|
+
try {
|
|
409
|
+
await this.#lifecycle.reset();
|
|
410
|
+
} catch (error) {
|
|
411
|
+
logNativeError('Full-text native reset failed', error);
|
|
412
|
+
throw new FullTextDerivedIndexError('Full-text native reset failed');
|
|
413
|
+
}
|
|
414
|
+
this.#assertSharedEpoch(ownerEpoch);
|
|
415
|
+
this.#activeEpoch = ownerEpoch;
|
|
416
|
+
this.#resetQueueState();
|
|
417
|
+
this.#unindexableRecords = 0;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
#boundedReset(operation: Promise<void>): Promise<void> {
|
|
421
|
+
return withTimeout(
|
|
422
|
+
operation,
|
|
423
|
+
this.#shutdownTimeoutMilliseconds,
|
|
424
|
+
() => new FullTextDerivedIndexError('Full-text native reset did not settle before timeout')
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
#clearResetOperation(operation: Promise<void>): void {
|
|
429
|
+
if (this.#resetOperation !== operation) return;
|
|
430
|
+
this.#resetOperation = undefined;
|
|
431
|
+
this.#resetEpoch = undefined;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
onStateChange(wake: (change?: DerivedIndexBackendStateChange) => void): () => void {
|
|
435
|
+
if (this.#wake && this.#wake !== wake)
|
|
436
|
+
throw new Error('Full-text derived index backend already has a state listener');
|
|
437
|
+
this.#wake = wake;
|
|
438
|
+
return () => {
|
|
439
|
+
if (this.#wake === wake) this.#wake = undefined;
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
#queueBarrier(epoch: bigint): void {
|
|
444
|
+
if (this.#lossPendingEpoch === epoch) return;
|
|
445
|
+
const horizon = this.#lastAcceptedSequence;
|
|
446
|
+
if (horizon === 0 || horizon <= this.#lastBarrierHorizon) return;
|
|
447
|
+
this.#lastBarrierHorizon = horizon;
|
|
448
|
+
this.#commands.push({
|
|
449
|
+
type: 'barrier',
|
|
450
|
+
epoch,
|
|
451
|
+
horizon,
|
|
452
|
+
cursor: cloneCursor(this.#lastAcceptedCursor ?? this.#durableCursor),
|
|
453
|
+
});
|
|
454
|
+
this.#scheduleDrain();
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
#scheduleDrain(): void {
|
|
458
|
+
if (this.#scheduled || this.#draining || ((this.#openRetryTimer || this.#writerRetryTimer) && !this.#shutdown))
|
|
459
|
+
return;
|
|
460
|
+
this.#scheduled = true;
|
|
461
|
+
setImmediate(() => {
|
|
462
|
+
this.#scheduled = false;
|
|
463
|
+
void this.#drain();
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
async #drain(): Promise<void> {
|
|
468
|
+
if (this.#draining) return;
|
|
469
|
+
this.#draining = true;
|
|
470
|
+
try {
|
|
471
|
+
while (this.#commands.length > 0 && !this.#failed) {
|
|
472
|
+
const next = this.#commands[0];
|
|
473
|
+
if ((next.type === 'barrier' || next.batch.records.length > 0) && !(await this.#ensureEngine())) {
|
|
474
|
+
if (this.#shutdown) this.#discardCommands();
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
if (this.#commands[0] !== next) continue;
|
|
478
|
+
if (next.type === 'apply') {
|
|
479
|
+
let complete = false;
|
|
480
|
+
try {
|
|
481
|
+
complete = await this.#apply(next);
|
|
482
|
+
} finally {
|
|
483
|
+
if (complete) {
|
|
484
|
+
this.#commands.shift();
|
|
485
|
+
this.#retainedBatches = Math.max(0, this.#retainedBatches - 1);
|
|
486
|
+
this.#retainedBytes = Math.max(0, this.#retainedBytes - next.bytes);
|
|
487
|
+
if (this.#capacityDeferred && !this.#failed) this.#notify('changed');
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
if (!complete || next.batch.records.length > 0) break;
|
|
491
|
+
} else {
|
|
492
|
+
this.#commands.shift();
|
|
493
|
+
if (!(await this.#publish(next))) break;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
} catch (error) {
|
|
497
|
+
this.#failAndNotify(error);
|
|
498
|
+
} finally {
|
|
499
|
+
this.#draining = false;
|
|
500
|
+
if (!this.#failed && !this.#openRetryTimer && !this.#writerRetryTimer && this.#commands.length > 0)
|
|
501
|
+
this.#scheduleDrain();
|
|
502
|
+
else if (this.#shutdown && !this.#shutdown.closing) void this.#closeForShutdown(this.#shutdown);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
async #ensureEngine(): Promise<boolean> {
|
|
507
|
+
if (this.#engine) return true;
|
|
508
|
+
const epoch = this.#activeEpoch;
|
|
509
|
+
if (epoch === undefined) throw new FullTextDerivedIndexError('Full-text owner epoch is unavailable');
|
|
510
|
+
const engine = await this.#open(epoch);
|
|
511
|
+
if (!engine) return false;
|
|
512
|
+
let actual: DerivedIndexCursor | undefined;
|
|
513
|
+
try {
|
|
514
|
+
this.#assertCommandEpoch(epoch);
|
|
515
|
+
actual = decodeFullTextCursorPayload(engine.committedPayload);
|
|
516
|
+
} catch (error) {
|
|
517
|
+
await this.#closeUninstalledEngine(engine);
|
|
518
|
+
throw error;
|
|
519
|
+
}
|
|
520
|
+
if (!sameCursor(actual, this.#durableCursor)) {
|
|
521
|
+
this.#lossPendingEpoch = epoch;
|
|
522
|
+
await this.#closeUninstalledEngine(engine);
|
|
523
|
+
this.#durableCursor = actual;
|
|
524
|
+
this.#cursorInspectedForAcquisition = true;
|
|
525
|
+
this.#resetWriterRetryBackoff();
|
|
526
|
+
this.#discardCommands();
|
|
527
|
+
this.#rewindAcceptedWork();
|
|
528
|
+
this.#notify('accepted-work-lost');
|
|
529
|
+
return false;
|
|
530
|
+
}
|
|
531
|
+
this.#engine = engine;
|
|
532
|
+
return true;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
async #closeUninstalledEngine(engine: FullTextDerivedIndexEngine): Promise<void> {
|
|
536
|
+
this.#settlingWriter = true;
|
|
537
|
+
try {
|
|
538
|
+
await this.#closeEngine(engine, { mode: 'rollback' });
|
|
539
|
+
} catch (error) {
|
|
540
|
+
this.#engine = engine;
|
|
541
|
+
logNativeError('Full-text writer could not close after lazy open failed', error);
|
|
542
|
+
throw new FullTextDerivedIndexError('Full-text writer could not close after lazy open failed');
|
|
543
|
+
} finally {
|
|
544
|
+
this.#settlingWriter = false;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
async #apply(command: ApplyCommand): Promise<boolean> {
|
|
549
|
+
this.#assertCommandEpoch(command.epoch);
|
|
550
|
+
if (command.batch.records.length === 0) {
|
|
551
|
+
this.#lastAppliedSequence = command.sequence;
|
|
552
|
+
return true;
|
|
553
|
+
}
|
|
554
|
+
const slice = toFullTextMutationSlice(
|
|
555
|
+
command.batch.records,
|
|
556
|
+
command.position,
|
|
557
|
+
this.#maxApplySliceRecords,
|
|
558
|
+
performance.now() + APPLY_SLICE_MILLISECONDS
|
|
559
|
+
);
|
|
560
|
+
const { batch: logical, end } = slice;
|
|
561
|
+
const mutationCount = logical.upserts.length + logical.deletes.length;
|
|
562
|
+
if (mutationCount > 0) {
|
|
563
|
+
this.#hasStagedMutations = true;
|
|
564
|
+
try {
|
|
565
|
+
const result = await this.#engine!.applyMutationBatch(logical, {
|
|
566
|
+
// A resolved derived-index chunk contains at most one mutation per source record key.
|
|
567
|
+
assumeDistinctIds: true,
|
|
568
|
+
rejectedUpsert: 'delete',
|
|
569
|
+
});
|
|
570
|
+
this.#recordApplyResult(result, mutationCount);
|
|
571
|
+
} catch (error) {
|
|
572
|
+
const permanentFailure = error instanceof FullTextDerivedIndexProtocolError;
|
|
573
|
+
await this.#loseAcceptedWork(command.epoch, error, {
|
|
574
|
+
notify: !permanentFailure,
|
|
575
|
+
retry: !permanentFailure,
|
|
576
|
+
});
|
|
577
|
+
if (permanentFailure) this.#failAndNotify(error);
|
|
578
|
+
return false;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
this.#assertCommandEpoch(command.epoch);
|
|
582
|
+
command.position = end;
|
|
583
|
+
if (command.position < command.batch.records.length) return false;
|
|
584
|
+
this.#lastAppliedSequence = command.sequence;
|
|
585
|
+
return true;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
#recordApplyResult(
|
|
589
|
+
result: Awaited<ReturnType<FullTextDerivedIndexEngine['applyMutationBatch']>>,
|
|
590
|
+
mutationCount: number
|
|
591
|
+
): void {
|
|
592
|
+
if (
|
|
593
|
+
!result ||
|
|
594
|
+
typeof result !== 'object' ||
|
|
595
|
+
!Number.isSafeInteger(result.processed) ||
|
|
596
|
+
result.processed !== mutationCount ||
|
|
597
|
+
!Array.isArray(result.rejected)
|
|
598
|
+
)
|
|
599
|
+
throw new FullTextDerivedIndexProtocolError('Full-text engine returned an invalid mutation result');
|
|
600
|
+
const rejected = result.rejected.length;
|
|
601
|
+
this.#stagedUnindexableRecords += rejected;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
async #publish(command: BarrierCommand): Promise<boolean> {
|
|
605
|
+
if (command.horizon <= this.#lastPublishedSequence) return true;
|
|
606
|
+
this.#assertCommandEpoch(command.epoch);
|
|
607
|
+
if (this.#lastAppliedSequence < command.horizon)
|
|
608
|
+
throw new FullTextDerivedIndexError('Full-text publication barrier passed unapplied work');
|
|
609
|
+
const cursor = withDurableCoverage(command.cursor ?? this.#durableCursor, this.#durableCursor);
|
|
610
|
+
try {
|
|
611
|
+
const payload = encodeFullTextCursorPayload(cursor, this.#maxCursorPayloadBytes);
|
|
612
|
+
await this.#engine!.publish(payload);
|
|
613
|
+
} catch (error) {
|
|
614
|
+
const terminal = error instanceof FullTextDerivedIndexConfigurationError;
|
|
615
|
+
if (terminal) {
|
|
616
|
+
this.#terminalFailure = error;
|
|
617
|
+
logError('Full-text derived index cannot publish its durable cursor', error);
|
|
618
|
+
}
|
|
619
|
+
await this.#loseAcceptedWork(command.epoch, error, { retry: !terminal });
|
|
620
|
+
return false;
|
|
621
|
+
}
|
|
622
|
+
this.#assertCommandEpoch(command.epoch);
|
|
623
|
+
this.#lastPublishedSequence = command.horizon;
|
|
624
|
+
this.#hasStagedMutations = false;
|
|
625
|
+
const rejected = this.#stagedUnindexableRecords;
|
|
626
|
+
this.#unindexableRecords += rejected;
|
|
627
|
+
this.#stagedUnindexableRecords = 0;
|
|
628
|
+
if (rejected > 0)
|
|
629
|
+
logWarning(
|
|
630
|
+
`Full-text derived index '${this.id}' removed ${rejected} records rejected by native record or frame limits`
|
|
631
|
+
);
|
|
632
|
+
this.#durableCursor = cloneCursor(cursor);
|
|
633
|
+
this.#cursorInspectedForAcquisition = true;
|
|
634
|
+
this.#resetWriterRetryBackoff();
|
|
635
|
+
if (!this.#shutdown) this.#notify('changed');
|
|
636
|
+
return true;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
async #loseAcceptedWork(
|
|
640
|
+
ownerEpoch: bigint,
|
|
641
|
+
cause: unknown,
|
|
642
|
+
options: { notify?: boolean; retry?: boolean } = {}
|
|
643
|
+
): Promise<void> {
|
|
644
|
+
this.#lossPendingEpoch = ownerEpoch;
|
|
645
|
+
this.#discardCommands();
|
|
646
|
+
const engine = this.#engine;
|
|
647
|
+
this.#engine = undefined;
|
|
648
|
+
if (!engine) {
|
|
649
|
+
logNativeError('Full-text writer is unavailable', cause);
|
|
650
|
+
throw new FullTextDerivedIndexError('Full-text writer is unavailable');
|
|
651
|
+
}
|
|
652
|
+
try {
|
|
653
|
+
await this.#closeEngine(engine, { mode: 'rollback' });
|
|
654
|
+
} catch (error) {
|
|
655
|
+
this.#engine = engine;
|
|
656
|
+
logNativeError('Full-text writer could not prove quiescence after losing accepted work', error);
|
|
657
|
+
throw new FullTextDerivedIndexError('Full-text writer could not prove quiescence after losing accepted work');
|
|
658
|
+
}
|
|
659
|
+
this.#rewindAcceptedWork();
|
|
660
|
+
this.#assertCommandEpoch(ownerEpoch);
|
|
661
|
+
this.#hasStagedMutations = false;
|
|
662
|
+
this.#stagedUnindexableRecords = 0;
|
|
663
|
+
if (options.notify === false) {
|
|
664
|
+
this.#lossPendingEpoch = undefined;
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
if (options.retry !== false) this.#scheduleWriterRetry(ownerEpoch, cause);
|
|
668
|
+
this.#notify('accepted-work-lost');
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
async #closeForShutdown(request: ShutdownRequest): Promise<void> {
|
|
672
|
+
if (request !== this.#shutdown || request.closing) return;
|
|
673
|
+
request.closing = true;
|
|
674
|
+
const engine = this.#engine;
|
|
675
|
+
try {
|
|
676
|
+
this.#clearOpenRetry();
|
|
677
|
+
this.#clearWriterRetry();
|
|
678
|
+
if (engine) {
|
|
679
|
+
const mode =
|
|
680
|
+
this.#hasStagedMutations || this.#lastAppliedSequence > this.#lastPublishedSequence
|
|
681
|
+
? 'rollback'
|
|
682
|
+
: 'require-clean';
|
|
683
|
+
await this.#closeEngine(engine, { mode });
|
|
684
|
+
}
|
|
685
|
+
this.#engine = undefined;
|
|
686
|
+
this.#activeEpoch = undefined;
|
|
687
|
+
this.#cursorInspectedForAcquisition = false;
|
|
688
|
+
this.#failed = false;
|
|
689
|
+
this.#shutdownFailure = undefined;
|
|
690
|
+
this.#resetQueueState();
|
|
691
|
+
if (this.#shutdown === request) this.#shutdown = undefined;
|
|
692
|
+
request.resolve();
|
|
693
|
+
} catch (error) {
|
|
694
|
+
if (this.#shutdown === request) this.#shutdown = undefined;
|
|
695
|
+
logNativeError('Full-text writer shutdown did not prove quiescence', error);
|
|
696
|
+
const failure = new FullTextDerivedIndexError('Full-text writer shutdown did not prove quiescence');
|
|
697
|
+
if (this.#markFailed(failure)) this.#shutdownFailure = failure;
|
|
698
|
+
request.reject(failure);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
async #closeEngine(
|
|
703
|
+
engine: FullTextDerivedIndexEngine,
|
|
704
|
+
options: { mode: 'require-clean' | 'rollback' }
|
|
705
|
+
): Promise<void> {
|
|
706
|
+
let closeOperation = this.#closeOperations.get(engine);
|
|
707
|
+
if (!closeOperation) {
|
|
708
|
+
closeOperation = engine.close(options);
|
|
709
|
+
this.#closeOperations.set(engine, closeOperation);
|
|
710
|
+
const installed = closeOperation;
|
|
711
|
+
void installed.catch(() => {
|
|
712
|
+
if (this.#closeOperations.get(engine) === installed) this.#closeOperations.delete(engine);
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
const result = await withTimeout(
|
|
716
|
+
closeOperation,
|
|
717
|
+
this.#closeTimeoutMilliseconds,
|
|
718
|
+
() => new FullTextDerivedIndexError('Full-text native writer close timed out')
|
|
719
|
+
);
|
|
720
|
+
if (result.cleanupError)
|
|
721
|
+
logNativeWarning(`Full-text derived index '${this.id}' closed with a native cleanup error`, result.cleanupError);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
async #open(ownerEpoch: bigint): Promise<FullTextDerivedIndexEngine | undefined> {
|
|
725
|
+
let lastError: unknown;
|
|
726
|
+
for (let attempt = 1; attempt <= this.#openAttempts; attempt++) {
|
|
727
|
+
this.#assertSharedEpoch(ownerEpoch);
|
|
728
|
+
try {
|
|
729
|
+
const engine = await this.#lifecycle.open();
|
|
730
|
+
this.#resetOpenRetryBackoff();
|
|
731
|
+
return engine;
|
|
732
|
+
} catch (error) {
|
|
733
|
+
lastError = error;
|
|
734
|
+
if (isTerminalOpenError(error)) {
|
|
735
|
+
logNativeError('Full-text writer could not be opened', error);
|
|
736
|
+
throw new FullTextDerivedIndexError('Full-text writer could not be opened');
|
|
737
|
+
}
|
|
738
|
+
if (attempt < this.#openAttempts && this.#openRetryMilliseconds > 0) {
|
|
739
|
+
await delay(this.#openRetryMilliseconds);
|
|
740
|
+
continue;
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
this.#scheduleOpenRetry(ownerEpoch, lastError);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
#scheduleOpenRetry(ownerEpoch: bigint, error: unknown): void {
|
|
748
|
+
if (this.#openRetryTimer || this.#shutdown || this.#failed) return;
|
|
749
|
+
const retryDelay = this.#openRetryDelayMilliseconds;
|
|
750
|
+
this.#openRetryDelayMilliseconds = Math.min(this.#maxOpenRetryMilliseconds, retryDelay * 2);
|
|
751
|
+
if (retryDelay >= this.#maxOpenRetryMilliseconds && !this.#openRetryWarned) {
|
|
752
|
+
this.#openRetryWarned = true;
|
|
753
|
+
logNativeWarning(`Full-text derived index '${this.id}' is waiting for its native writer`, error);
|
|
754
|
+
}
|
|
755
|
+
this.#openRetryTimer = setTimeout(() => {
|
|
756
|
+
this.#openRetryTimer = undefined;
|
|
757
|
+
if (this.#activeEpoch === ownerEpoch && this.#host?.isOwnerEpoch(ownerEpoch)) this.#scheduleDrain();
|
|
758
|
+
}, retryDelay);
|
|
759
|
+
this.#openRetryTimer.unref?.();
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
#clearOpenRetry(): void {
|
|
763
|
+
if (!this.#openRetryTimer) return;
|
|
764
|
+
clearTimeout(this.#openRetryTimer);
|
|
765
|
+
this.#openRetryTimer = undefined;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
#resetOpenRetryBackoff(): void {
|
|
769
|
+
this.#clearOpenRetry();
|
|
770
|
+
this.#openRetryDelayMilliseconds = Math.max(1, this.#openRetryMilliseconds);
|
|
771
|
+
this.#openRetryWarned = false;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
#scheduleWriterRetry(ownerEpoch: bigint, error: unknown): void {
|
|
775
|
+
if (this.#writerRetryTimer || this.#shutdown || this.#failed || this.#terminalFailure) return;
|
|
776
|
+
const retryDelay = this.#writerRetryDelayMilliseconds;
|
|
777
|
+
this.#writerRetryDelayMilliseconds = Math.min(
|
|
778
|
+
this.#maxOpenRetryMilliseconds,
|
|
779
|
+
this.#writerRetryDelayMilliseconds * 2
|
|
780
|
+
);
|
|
781
|
+
if (retryDelay >= this.#maxOpenRetryMilliseconds && !this.#writerRetryWarned) {
|
|
782
|
+
this.#writerRetryWarned = true;
|
|
783
|
+
logNativeWarning(`Full-text derived index '${this.id}' is retrying native writer work`, error);
|
|
784
|
+
}
|
|
785
|
+
this.#writerRetryTimer = setTimeout(() => {
|
|
786
|
+
this.#writerRetryTimer = undefined;
|
|
787
|
+
if (this.#activeEpoch === ownerEpoch && this.#host?.isOwnerEpoch(ownerEpoch)) this.#notify('changed');
|
|
788
|
+
}, retryDelay);
|
|
789
|
+
this.#writerRetryTimer.unref?.();
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
#clearWriterRetry(): void {
|
|
793
|
+
if (!this.#writerRetryTimer) return;
|
|
794
|
+
clearTimeout(this.#writerRetryTimer);
|
|
795
|
+
this.#writerRetryTimer = undefined;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
#resetWriterRetryBackoff(): void {
|
|
799
|
+
this.#clearWriterRetry();
|
|
800
|
+
this.#writerRetryDelayMilliseconds = Math.min(
|
|
801
|
+
this.#maxOpenRetryMilliseconds,
|
|
802
|
+
Math.max(DEFAULT_OPEN_RETRY_MILLISECONDS, this.#openRetryMilliseconds)
|
|
803
|
+
);
|
|
804
|
+
this.#writerRetryWarned = false;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
#discardCommands(): void {
|
|
808
|
+
this.#commands = [];
|
|
809
|
+
this.#retainedBatches = 0;
|
|
810
|
+
this.#retainedBytes = 0;
|
|
811
|
+
this.#lastBarrierHorizon = this.#lastPublishedSequence;
|
|
812
|
+
this.#capacityDeferred = false;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
#rewindAcceptedWork(): void {
|
|
816
|
+
this.#lastAcceptedSequence = this.#lastPublishedSequence;
|
|
817
|
+
this.#lastAppliedSequence = this.#lastPublishedSequence;
|
|
818
|
+
this.#lastBarrierHorizon = this.#lastPublishedSequence;
|
|
819
|
+
this.#lastAcceptedCursor = cloneCursor(this.#durableCursor);
|
|
820
|
+
this.#hasStagedMutations = false;
|
|
821
|
+
this.#stagedUnindexableRecords = 0;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
#resetQueueState(): void {
|
|
825
|
+
this.#resetOpenRetryBackoff();
|
|
826
|
+
this.#resetWriterRetryBackoff();
|
|
827
|
+
this.#commands = [];
|
|
828
|
+
this.#retainedBatches = 0;
|
|
829
|
+
this.#retainedBytes = 0;
|
|
830
|
+
this.#lastAcceptedSequence = 0;
|
|
831
|
+
this.#lastAppliedSequence = 0;
|
|
832
|
+
this.#lastPublishedSequence = 0;
|
|
833
|
+
this.#lastBarrierHorizon = 0;
|
|
834
|
+
this.#lastAcceptedCursor = undefined;
|
|
835
|
+
this.#hasStagedMutations = false;
|
|
836
|
+
this.#stagedUnindexableRecords = 0;
|
|
837
|
+
this.#capacityDeferred = false;
|
|
838
|
+
this.#lossPendingEpoch = undefined;
|
|
839
|
+
this.#invalidEstimateWarned = false;
|
|
840
|
+
this.#failedDeliveryObserved = false;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
#markFailed(error: unknown): boolean {
|
|
844
|
+
if (this.#failed) return false;
|
|
845
|
+
this.#failed = true;
|
|
846
|
+
this.#clearOpenRetry();
|
|
847
|
+
this.#failedDeliveryObserved = false;
|
|
848
|
+
this.#discardCommands();
|
|
849
|
+
logError('Full-text derived index backend failed', error);
|
|
850
|
+
return true;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
#failAndNotify(error: unknown): void {
|
|
854
|
+
if (this.#markFailed(error)) this.#notify('failed');
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
#assertAttached(): void {
|
|
858
|
+
if (!this.#host) throw new Error('Full-text derived index backend must be attached before use');
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
#assertSharedEpoch(ownerEpoch: bigint): void {
|
|
862
|
+
if (!this.#host?.isOwnerEpoch(ownerEpoch)) throw new FullTextDerivedIndexError('Full-text owner epoch was revoked');
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
#assertCommandEpoch(ownerEpoch: bigint): void {
|
|
866
|
+
if (this.#activeEpoch !== ownerEpoch || !this.#host?.isOwnerEpoch(ownerEpoch))
|
|
867
|
+
throw new FullTextDerivedIndexError('Full-text owner epoch was revoked');
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
#notify(change: DerivedIndexBackendStateChange): void {
|
|
871
|
+
if (change === 'changed') this.#capacityDeferred = false;
|
|
872
|
+
const wake = this.#wake;
|
|
873
|
+
const activeEpoch = this.#activeEpoch;
|
|
874
|
+
if (!wake) {
|
|
875
|
+
if (change === 'accepted-work-lost' && this.#lossPendingEpoch === activeEpoch) this.#lossPendingEpoch = undefined;
|
|
876
|
+
return;
|
|
877
|
+
}
|
|
878
|
+
setImmediate(() => {
|
|
879
|
+
if (change === 'failed' && this.#failedDeliveryObserved) return;
|
|
880
|
+
if (this.#activeEpoch !== activeEpoch || this.#wake !== wake) {
|
|
881
|
+
if (change === 'accepted-work-lost' && this.#lossPendingEpoch === activeEpoch)
|
|
882
|
+
this.#lossPendingEpoch = undefined;
|
|
883
|
+
return;
|
|
884
|
+
}
|
|
885
|
+
try {
|
|
886
|
+
wake(change);
|
|
887
|
+
} catch (error) {
|
|
888
|
+
logError('Full-text derived index state notification failed', error);
|
|
889
|
+
} finally {
|
|
890
|
+
if (change === 'accepted-work-lost' && this.#lossPendingEpoch === activeEpoch)
|
|
891
|
+
this.#lossPendingEpoch = undefined;
|
|
892
|
+
}
|
|
893
|
+
});
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
export function toFullTextMutationBatch(batch: DerivedIndexBatch): FullTextMutationBatch {
|
|
898
|
+
return toFullTextMutationSlice(batch.records, 0, batch.records.length, Number.POSITIVE_INFINITY).batch;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
function toFullTextMutationSlice(
|
|
902
|
+
records: DerivedIndexBatch['records'],
|
|
903
|
+
start: number,
|
|
904
|
+
maxRecords: number,
|
|
905
|
+
deadline: number
|
|
906
|
+
): { batch: FullTextMutationBatch; end: number } {
|
|
907
|
+
const upserts: FullTextMutationBatch['upserts'] = [];
|
|
908
|
+
const deletes: string[] = [];
|
|
909
|
+
const limit = Math.min(start + maxRecords, records.length);
|
|
910
|
+
let index = start;
|
|
911
|
+
let previousTableId: number | undefined;
|
|
912
|
+
let tablePrefix = '';
|
|
913
|
+
for (; index < limit; index++) {
|
|
914
|
+
const record = records[index];
|
|
915
|
+
if (record.recordId == null || typeof record.recordId === 'symbol') continue;
|
|
916
|
+
if (record.tableId !== previousTableId) {
|
|
917
|
+
previousTableId = record.tableId;
|
|
918
|
+
tablePrefix = `${record.tableId}.`;
|
|
919
|
+
}
|
|
920
|
+
const id = tablePrefix + toBufferKey(record.recordId).toString('base64url');
|
|
921
|
+
const fields = record.state.kind === 'record' ? fullTextFields(record.state.projection) : undefined;
|
|
922
|
+
if (fields) upserts.push({ id, fields });
|
|
923
|
+
else deletes.push(id);
|
|
924
|
+
if ((index - start + 1) % 16 === 0 && performance.now() >= deadline) {
|
|
925
|
+
index++;
|
|
926
|
+
break;
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
return { batch: { upserts, deletes }, end: index };
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
function logWarning(message: string, error?: unknown): void {
|
|
933
|
+
log('warn', message, error);
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
function logError(message: string, error?: unknown): void {
|
|
937
|
+
log('error', message, error);
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
function logNativeWarning(message: string, error?: unknown): void {
|
|
941
|
+
log('warn', message, error, true);
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
function logNativeError(message: string, error?: unknown): void {
|
|
945
|
+
log('error', message, error, true);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
function log(level: 'warn' | 'error', message: string, error?: unknown, native = false): void {
|
|
949
|
+
const code = nativeErrorCode(error);
|
|
950
|
+
const detail = error instanceof FullTextDerivedIndexError ? error.message : code;
|
|
951
|
+
const name = error instanceof Error ? error.name : undefined;
|
|
952
|
+
const summary = `${message}${detail || name ? ` (${detail ?? name}${code && detail !== code ? `; ${code}` : ''})` : ''}`;
|
|
953
|
+
try {
|
|
954
|
+
if (!native && error instanceof Error && code === undefined && !(error instanceof FullTextDerivedIndexError))
|
|
955
|
+
logger[level]?.(summary, error);
|
|
956
|
+
else logger[level]?.(summary);
|
|
957
|
+
} catch {}
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
function nativeErrorCode(error: unknown): string | undefined {
|
|
961
|
+
return findNativeErrorCode(error, new Set());
|
|
962
|
+
function findNativeErrorCode(value: unknown, seen: Set<object>): string | undefined {
|
|
963
|
+
if (!value || typeof value !== 'object' || seen.has(value)) return;
|
|
964
|
+
seen.add(value);
|
|
965
|
+
if ('code' in value && typeof value.code === 'string') return value.code;
|
|
966
|
+
if (value instanceof AggregateError) {
|
|
967
|
+
for (const nested of value.errors) {
|
|
968
|
+
const code = findNativeErrorCode(nested, seen);
|
|
969
|
+
if (code) return code;
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
if ('cause' in value) return findNativeErrorCode(value.cause, seen);
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
export function encodeFullTextCursorPayload(
|
|
977
|
+
cursor: DerivedIndexCursor | undefined,
|
|
978
|
+
maxBytes = HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES
|
|
979
|
+
): string {
|
|
980
|
+
const normalized = cursor && normalizedCursor(cursor);
|
|
981
|
+
const payload = JSON.stringify({
|
|
982
|
+
format: 1,
|
|
983
|
+
cursor: normalized
|
|
984
|
+
? {
|
|
985
|
+
format: 1,
|
|
986
|
+
logs: Object.fromEntries(
|
|
987
|
+
Object.entries(normalized.logs).sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
|
988
|
+
),
|
|
989
|
+
...(normalized.coverage ? { coverage: sortedPositions(normalized.coverage) } : null),
|
|
990
|
+
}
|
|
991
|
+
: null,
|
|
992
|
+
});
|
|
993
|
+
if (Buffer.byteLength(payload) > maxBytes)
|
|
994
|
+
throw new FullTextDerivedIndexConfigurationError(
|
|
995
|
+
`Full-text cursor payload exceeds the configured ${maxBytes}-byte limit`
|
|
996
|
+
);
|
|
997
|
+
return payload;
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
export function decodeFullTextCursorPayload(
|
|
1001
|
+
payload: string | null | undefined,
|
|
1002
|
+
maxBytes = HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES
|
|
1003
|
+
): DerivedIndexCursor | undefined {
|
|
1004
|
+
if (payload == null) return;
|
|
1005
|
+
if (typeof payload !== 'string' || Buffer.byteLength(payload) > maxBytes)
|
|
1006
|
+
throw new FullTextDerivedIndexError('Full-text cursor payload is invalid or too large');
|
|
1007
|
+
let decoded: unknown;
|
|
1008
|
+
try {
|
|
1009
|
+
decoded = JSON.parse(payload);
|
|
1010
|
+
} catch {
|
|
1011
|
+
throw new FullTextDerivedIndexError('Full-text cursor payload is not valid JSON');
|
|
1012
|
+
}
|
|
1013
|
+
if (
|
|
1014
|
+
!plainObject(decoded) ||
|
|
1015
|
+
decoded.format !== 1 ||
|
|
1016
|
+
!Object.hasOwn(decoded, 'cursor') ||
|
|
1017
|
+
Object.keys(decoded).some((name) => name !== 'format' && name !== 'cursor')
|
|
1018
|
+
)
|
|
1019
|
+
throw new FullTextDerivedIndexError('Full-text cursor payload has an unsupported format');
|
|
1020
|
+
if (decoded.cursor === null) return;
|
|
1021
|
+
if (
|
|
1022
|
+
!plainObject(decoded.cursor) ||
|
|
1023
|
+
decoded.cursor.format !== 1 ||
|
|
1024
|
+
!plainObject(decoded.cursor.logs) ||
|
|
1025
|
+
Object.keys(decoded.cursor).some((name) => name !== 'format' && name !== 'logs' && name !== 'coverage')
|
|
1026
|
+
)
|
|
1027
|
+
throw new FullTextDerivedIndexError('Full-text cursor payload contains an invalid cursor');
|
|
1028
|
+
return normalizedCursor(decoded.cursor as DerivedIndexCursor);
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
function fullTextFields(projection: unknown): Record<string, string | string[]> | undefined {
|
|
1032
|
+
if (!projection || typeof projection !== 'object' || Array.isArray(projection)) return;
|
|
1033
|
+
let fields: Record<string, string | string[]> | undefined;
|
|
1034
|
+
for (const name in projection) {
|
|
1035
|
+
if (!Object.hasOwn(projection, name)) continue;
|
|
1036
|
+
const value = (projection as Record<string, unknown>)[name];
|
|
1037
|
+
if (typeof value === 'string') (fields ??= Object.create(null))[name] = value;
|
|
1038
|
+
else if (Array.isArray(value)) (fields ??= Object.create(null))[name] = value as string[];
|
|
1039
|
+
}
|
|
1040
|
+
return fields;
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function isTerminalOpenError(error: unknown): boolean {
|
|
1044
|
+
const code = nativeErrorCode(error);
|
|
1045
|
+
return code !== undefined && TERMINAL_OPEN_ERROR_CODES.has(code);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
function cloneCursor(cursor: DerivedIndexCursor | undefined): DerivedIndexCursor | undefined {
|
|
1049
|
+
if (!cursor) return;
|
|
1050
|
+
const logs = Object.freeze({ ...cursor.logs });
|
|
1051
|
+
const coverage = cursor.coverage && clonePositions(cursor.coverage);
|
|
1052
|
+
return Object.freeze({ format: 1, logs, ...(coverage ? { coverage } : null) });
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function withDurableCoverage(
|
|
1056
|
+
cursor: DerivedIndexCursor | undefined,
|
|
1057
|
+
durable: DerivedIndexCursor | undefined
|
|
1058
|
+
): DerivedIndexCursor | undefined {
|
|
1059
|
+
if (!cursor || cursor.coverage || !durable?.coverage) return cursor;
|
|
1060
|
+
return { ...cursor, coverage: durable.coverage };
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
function sameCursor(left: DerivedIndexCursor | undefined, right: DerivedIndexCursor | undefined): boolean {
|
|
1064
|
+
if (!left || !right) return left === right;
|
|
1065
|
+
const leftLogs = Object.entries(left.logs);
|
|
1066
|
+
const rightLogs = Object.entries(right.logs);
|
|
1067
|
+
if (leftLogs.length !== rightLogs.length) return false;
|
|
1068
|
+
if (!leftLogs.every(([name, timestamp]) => Object.hasOwn(right.logs, name) && right.logs[name] === timestamp))
|
|
1069
|
+
return false;
|
|
1070
|
+
if (!left.coverage || !right.coverage) return left.coverage === right.coverage;
|
|
1071
|
+
return sameDerivedIndexPositions(left.coverage, right.coverage);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
function normalizedCursor(cursor: DerivedIndexCursor): DerivedIndexCursor {
|
|
1075
|
+
if (!plainObject(cursor) || cursor.format !== 1 || !plainObject(cursor.logs))
|
|
1076
|
+
throw new FullTextDerivedIndexError('Full-text cursor payload contains an invalid cursor');
|
|
1077
|
+
const logs: Record<string, number> = Object.create(null);
|
|
1078
|
+
for (const name in cursor.logs) {
|
|
1079
|
+
if (!Object.hasOwn(cursor.logs, name)) continue;
|
|
1080
|
+
const timestamp = cursor.logs[name];
|
|
1081
|
+
if (!name || !Number.isFinite(timestamp) || timestamp <= 0)
|
|
1082
|
+
throw new FullTextDerivedIndexError('Full-text cursor payload contains an invalid log position');
|
|
1083
|
+
logs[name] = timestamp;
|
|
1084
|
+
}
|
|
1085
|
+
let coverage: DerivedIndexPositions | undefined;
|
|
1086
|
+
if (cursor.coverage !== undefined) coverage = normalizedPositions(cursor.coverage);
|
|
1087
|
+
Object.freeze(logs);
|
|
1088
|
+
return Object.freeze({ format: 1, logs, ...(coverage ? { coverage } : null) });
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
function normalizedPositions(value: unknown): DerivedIndexPositions {
|
|
1092
|
+
if (!plainObject(value)) throw new FullTextDerivedIndexError('Full-text cursor payload contains invalid coverage');
|
|
1093
|
+
const positions: DerivedIndexPositions = Object.create(null);
|
|
1094
|
+
for (const name in value) {
|
|
1095
|
+
if (!Object.hasOwn(value, name)) continue;
|
|
1096
|
+
if (!name) throw new FullTextDerivedIndexError('Full-text cursor payload contains invalid coverage');
|
|
1097
|
+
const position = value[name];
|
|
1098
|
+
if (position === null) positions[name] = null;
|
|
1099
|
+
else if (
|
|
1100
|
+
plainObject(position) &&
|
|
1101
|
+
Object.keys(position).length === 2 &&
|
|
1102
|
+
Object.hasOwn(position, 'sequence') &&
|
|
1103
|
+
Object.hasOwn(position, 'offset') &&
|
|
1104
|
+
Number.isSafeInteger(position.sequence) &&
|
|
1105
|
+
position.sequence >= 0 &&
|
|
1106
|
+
Number.isSafeInteger(position.offset) &&
|
|
1107
|
+
position.offset >= 0
|
|
1108
|
+
)
|
|
1109
|
+
positions[name] = { sequence: position.sequence, offset: position.offset };
|
|
1110
|
+
else throw new FullTextDerivedIndexError('Full-text cursor payload contains invalid coverage');
|
|
1111
|
+
}
|
|
1112
|
+
return Object.freeze(positions);
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
function clonePositions(positions: DerivedIndexPositions): DerivedIndexPositions {
|
|
1116
|
+
const clone: DerivedIndexPositions = Object.create(null);
|
|
1117
|
+
for (const name in positions) {
|
|
1118
|
+
if (!Object.hasOwn(positions, name)) continue;
|
|
1119
|
+
const position = positions[name];
|
|
1120
|
+
clone[name] = position && { sequence: position.sequence, offset: position.offset };
|
|
1121
|
+
}
|
|
1122
|
+
return Object.freeze(clone);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
function sortedPositions(positions: DerivedIndexPositions): DerivedIndexPositions {
|
|
1126
|
+
return Object.fromEntries(
|
|
1127
|
+
Object.entries(positions)
|
|
1128
|
+
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
|
1129
|
+
.map(([name, position]) => [name, position && { sequence: position.sequence, offset: position.offset }])
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
function plainObject(value: unknown): value is Record<string, any> {
|
|
1134
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
|
1135
|
+
const prototype = Object.getPrototypeOf(value);
|
|
1136
|
+
return prototype === Object.prototype || prototype === null;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
function positiveInteger(value: number, name: string): number {
|
|
1140
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${name} must be a positive safe integer`);
|
|
1141
|
+
return value;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
function nonNegativeInteger(value: number, name: string): number {
|
|
1145
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${name} must be a non-negative safe integer`);
|
|
1146
|
+
return value;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
function delay(milliseconds: number): Promise<void> {
|
|
1150
|
+
return new Promise((resolve) => {
|
|
1151
|
+
const timer = setTimeout(resolve, milliseconds);
|
|
1152
|
+
timer.unref?.();
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
function withTimeout<T>(promise: Promise<T>, milliseconds: number, timeoutError: () => Error): Promise<T> {
|
|
1157
|
+
let timer: NodeJS.Timeout;
|
|
1158
|
+
return Promise.race([
|
|
1159
|
+
promise,
|
|
1160
|
+
new Promise<never>((_resolve, reject) => {
|
|
1161
|
+
timer = setTimeout(() => reject(timeoutError()), milliseconds);
|
|
1162
|
+
timer.unref?.();
|
|
1163
|
+
}),
|
|
1164
|
+
]).finally(() => clearTimeout(timer));
|
|
1165
|
+
}
|