@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cliOperations.ts +12 -0
- package/bin/restart.ts +66 -6
- package/components/Application.ts +1134 -112
- package/components/DESIGN.md +421 -0
- package/components/OptionsWatcher.ts +368 -102
- package/components/Scope.ts +20 -1
- package/components/componentLoader.ts +43 -4
- package/components/deploymentOperations.ts +4 -1
- package/components/deploymentRecorder.ts +9 -2
- package/components/mcp/DESIGN.md +109 -0
- package/components/mcp/audit.ts +21 -17
- package/components/operations.js +284 -52
- package/components/operationsValidation.js +49 -2
- package/components/packageComponent.ts +25 -1
- package/components/requestRestart.ts +11 -0
- package/config/DESIGN.md +306 -0
- package/config/RootConfigWatcher.ts +191 -37
- package/config/configReadRetry.ts +62 -0
- package/config/configUtils.ts +78 -26
- package/config/parseConfigFile.ts +34 -0
- package/config/readConfigFileSync.ts +44 -0
- package/config/watcherArming.ts +59 -0
- package/config-root.schema.json +4 -0
- package/dataLayer/DESIGN.md +179 -0
- package/dataLayer/harperBridge/ResourceBridge.ts +28 -2
- package/dataLayer/restoreMarker.ts +92 -25
- package/dist/bin/cliOperations.js +13 -0
- package/dist/bin/cliOperations.js.map +1 -1
- package/dist/bin/restart.js +42 -6
- package/dist/bin/restart.js.map +1 -1
- package/dist/components/Application.d.ts +104 -9
- package/dist/components/Application.js +954 -102
- package/dist/components/Application.js.map +1 -1
- package/dist/components/OptionsWatcher.d.ts +4 -1
- package/dist/components/OptionsWatcher.js +378 -104
- package/dist/components/OptionsWatcher.js.map +1 -1
- package/dist/components/Scope.js +15 -1
- package/dist/components/Scope.js.map +1 -1
- package/dist/components/componentLoader.js +35 -3
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/deploymentOperations.js +4 -1
- package/dist/components/deploymentOperations.js.map +1 -1
- package/dist/components/deploymentRecorder.d.ts +4 -2
- package/dist/components/deploymentRecorder.js +1 -0
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/mcp/audit.d.ts +2 -1
- package/dist/components/mcp/audit.js +21 -17
- package/dist/components/mcp/audit.js.map +1 -1
- package/dist/components/operations.d.ts +28 -0
- package/dist/components/operations.js +263 -54
- package/dist/components/operations.js.map +1 -1
- package/dist/components/operationsValidation.js +48 -2
- package/dist/components/operationsValidation.js.map +1 -1
- package/dist/components/packageComponent.js +24 -0
- package/dist/components/packageComponent.js.map +1 -1
- package/dist/components/requestRestart.d.ts +1 -0
- package/dist/components/requestRestart.js +7 -0
- package/dist/components/requestRestart.js.map +1 -1
- package/dist/config/RootConfigWatcher.d.ts +2 -0
- package/dist/config/RootConfigWatcher.js +189 -35
- package/dist/config/RootConfigWatcher.js.map +1 -1
- package/dist/config/configReadRetry.d.ts +8 -0
- package/dist/config/configReadRetry.js +62 -0
- package/dist/config/configReadRetry.js.map +1 -0
- package/dist/config/configUtils.d.ts +10 -9
- package/dist/config/configUtils.js +63 -27
- package/dist/config/configUtils.js.map +1 -1
- package/dist/config/parseConfigFile.d.ts +4 -0
- package/dist/config/parseConfigFile.js +35 -0
- package/dist/config/parseConfigFile.js.map +1 -0
- package/dist/config/readConfigFileSync.d.ts +1 -0
- package/dist/config/readConfigFileSync.js +47 -0
- package/dist/config/readConfigFileSync.js.map +1 -0
- package/dist/config/watcherArming.d.ts +15 -0
- package/dist/config/watcherArming.js +59 -0
- package/dist/config/watcherArming.js.map +1 -0
- package/dist/dataLayer/harperBridge/ResourceBridge.js +21 -2
- package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
- package/dist/dataLayer/restoreMarker.d.ts +21 -8
- package/dist/dataLayer/restoreMarker.js +94 -27
- package/dist/dataLayer/restoreMarker.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +3 -0
- package/dist/resources/DatabaseTransaction.d.ts +25 -0
- package/dist/resources/DatabaseTransaction.js +224 -18
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +2 -1
- package/dist/resources/LMDBTransaction.js +22 -3
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/PrimaryRocksDatabase.js +22 -6
- package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
- package/dist/resources/RecordEncoder.d.ts +1 -1
- package/dist/resources/RecordEncoder.js +46 -10
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/Resource.js +97 -13
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +8 -0
- package/dist/resources/RocksIndexStore.js +2 -1
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/RocksTransactionLogStore.d.ts +10 -0
- package/dist/resources/RocksTransactionLogStore.js +104 -33
- package/dist/resources/RocksTransactionLogStore.js.map +1 -1
- package/dist/resources/Table.d.ts +58 -7
- package/dist/resources/Table.js +1143 -348
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.d.ts +3 -0
- package/dist/resources/analytics/write.js +49 -16
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +170 -0
- package/dist/resources/auditStore.js +457 -11
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/crdt.d.ts +10 -0
- package/dist/resources/crdt.js +22 -0
- package/dist/resources/crdt.js.map +1 -1
- package/dist/resources/dataLoader.js +3 -4
- package/dist/resources/dataLoader.js.map +1 -1
- package/dist/resources/databases.d.ts +16 -13
- package/dist/resources/databases.js +625 -177
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/derivedIndexRegistry.d.ts +5 -0
- package/dist/resources/derivedIndexRegistry.js +68 -0
- package/dist/resources/derivedIndexRegistry.js.map +1 -0
- package/dist/resources/derivedIndexRuntime.d.ts +215 -0
- package/dist/resources/derivedIndexRuntime.js +2027 -0
- package/dist/resources/derivedIndexRuntime.js.map +1 -0
- package/dist/resources/graphql.js +3 -2
- package/dist/resources/graphql.js.map +1 -1
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +102 -9
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +867 -39
- package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
- package/dist/resources/indexes/fullTextDerivedIndex.d.ts +81 -0
- package/dist/resources/indexes/fullTextDerivedIndex.js +1004 -0
- package/dist/resources/indexes/fullTextDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/fullTextNativeBinding.d.ts +78 -0
- package/dist/resources/indexes/fullTextNativeBinding.js +85 -0
- package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -0
- package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
- package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
- package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
- package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
- package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
- package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
- package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.d.ts +24 -0
- package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js +149 -0
- package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js.map +1 -0
- package/dist/resources/nodeIdMapping.d.ts +5 -0
- package/dist/resources/nodeIdMapping.js +49 -0
- package/dist/resources/nodeIdMapping.js.map +1 -1
- package/dist/resources/recordLock.d.ts +47 -4
- package/dist/resources/recordLock.js +138 -7
- package/dist/resources/recordLock.js.map +1 -1
- package/dist/resources/recordLockCoordinator.d.ts +557 -0
- package/dist/resources/recordLockCoordinator.js +2597 -0
- package/dist/resources/recordLockCoordinator.js.map +1 -0
- package/dist/resources/replayLogs.js +5 -0
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/replicatedApplyFailure.d.ts +16 -0
- package/dist/resources/replicatedApplyFailure.js +63 -0
- package/dist/resources/replicatedApplyFailure.js.map +1 -0
- package/dist/resources/scheduler/scheduler.js +3 -3
- package/dist/resources/scheduler/scheduler.js.map +1 -1
- package/dist/resources/search.d.ts +10 -4
- package/dist/resources/search.js +160 -40
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/tracked.d.ts +5 -1
- package/dist/resources/tracked.js +74 -23
- package/dist/resources/tracked.js.map +1 -1
- package/dist/resources/transactionBroadcast.js +4 -6
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/auth.js +59 -23
- package/dist/security/auth.js.map +1 -1
- package/dist/security/deferredAuthentication.d.ts +11 -0
- package/dist/security/deferredAuthentication.js +25 -3
- package/dist/security/deferredAuthentication.js.map +1 -1
- package/dist/security/jsLoader.js +9 -4
- package/dist/security/jsLoader.js.map +1 -1
- package/dist/server/REST.js +39 -5
- package/dist/server/REST.js.map +1 -1
- package/dist/server/http.d.ts +5 -1
- package/dist/server/http.js +34 -2
- package/dist/server/http.js.map +1 -1
- package/dist/server/mqtt.js +5 -1
- package/dist/server/mqtt.js.map +1 -1
- package/dist/server/serverHelpers/Headers.d.ts +2 -0
- package/dist/server/serverHelpers/Headers.js +6 -0
- package/dist/server/serverHelpers/Headers.js.map +1 -1
- package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
- package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
- package/dist/server/serverHelpers/Request.d.ts +5 -10
- package/dist/server/serverHelpers/Request.js +38 -136
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.d.ts +2 -0
- package/dist/server/serverHelpers/contentTypes.js +189 -15
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/serverUtilities.d.ts +3 -3
- package/dist/server/serverHelpers/serverUtilities.js +110 -20
- package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.js +4 -1
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/serverHelpers/webSocketCloseReason.d.ts +2 -0
- package/dist/server/serverHelpers/webSocketCloseReason.js +29 -0
- package/dist/server/serverHelpers/webSocketCloseReason.js.map +1 -0
- package/dist/server/storageReclamation.js +1 -1
- package/dist/server/storageReclamation.js.map +1 -1
- package/dist/server/threads/isolatedApplications.d.ts +47 -0
- package/dist/server/threads/isolatedApplications.js +171 -0
- package/dist/server/threads/isolatedApplications.js.map +1 -0
- package/dist/server/threads/logRotationTransport.d.ts +1 -0
- package/dist/server/threads/logRotationTransport.js +33 -0
- package/dist/server/threads/logRotationTransport.js.map +1 -0
- package/dist/server/threads/manageThreads.d.ts +64 -6
- package/dist/server/threads/manageThreads.js +261 -12
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/socketRouter.d.ts +1 -0
- package/dist/server/threads/socketRouter.js +196 -13
- package/dist/server/threads/socketRouter.js.map +1 -1
- package/dist/server/threads/threadServer.js +30 -7
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/utility/errors/hdbError.d.ts +24 -0
- package/dist/utility/errors/hdbError.js +58 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/hdbTerms.d.ts +2 -0
- package/dist/utility/hdbTerms.js +2 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/utility/logging/harper_logger.js +218 -38
- package/dist/utility/logging/harper_logger.js.map +1 -1
- package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
- package/dist/utility/logging/logGenerationCoordinator.js +184 -0
- package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
- package/dist/utility/logging/logRotation.d.ts +46 -0
- package/dist/utility/logging/logRotation.js +365 -0
- package/dist/utility/logging/logRotation.js.map +1 -0
- package/dist/utility/logging/logRotator.d.ts +1 -1
- package/dist/utility/logging/logRotator.js +172 -92
- package/dist/utility/logging/logRotator.js.map +1 -1
- package/dist/utility/npmUtilities.js +6 -4
- package/dist/utility/npmUtilities.js.map +1 -1
- package/dist/utility/watcherFallback.d.ts +0 -45
- package/dist/utility/watcherFallback.js +1 -125
- package/dist/utility/watcherFallback.js.map +1 -1
- package/dist/validation/configValidator.js +6 -3
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +9 -0
- package/json/systemSchema.json +3 -0
- package/npm-shrinkwrap.json +177 -87
- package/package.json +13 -5
- package/resources/DESIGN.md +689 -19
- package/resources/DatabaseTransaction.ts +230 -17
- package/resources/LMDBTransaction.ts +21 -3
- package/resources/PrimaryRocksDatabase.ts +20 -7
- package/resources/RecordEncoder.ts +52 -10
- package/resources/Resource.ts +97 -13
- package/resources/ResourceInterface.ts +8 -0
- package/resources/RocksIndexStore.ts +2 -1
- package/resources/RocksTransactionLogStore.ts +111 -31
- package/resources/Table.ts +1273 -396
- package/resources/analytics/DESIGN.md +38 -0
- package/resources/analytics/write.ts +50 -17
- package/resources/auditStore.ts +460 -11
- package/resources/crdt.ts +22 -0
- package/resources/dataLoader.ts +3 -4
- package/resources/databases.ts +611 -147
- package/resources/derivedIndexRegistry.ts +56 -0
- package/resources/derivedIndexRuntime.ts +2292 -0
- package/resources/graphql.ts +3 -2
- package/resources/indexes/DESIGN.md +833 -0
- package/resources/indexes/HierarchicalNavigableSmallWorld.ts +924 -46
- package/resources/indexes/fullTextDerivedIndex.ts +1165 -0
- package/resources/indexes/fullTextNativeBinding.ts +146 -0
- package/resources/indexes/hnswDerivedIndex.ts +531 -0
- package/resources/indexes/hnswPlaneBinding.ts +174 -0
- package/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts +181 -0
- package/resources/nodeIdMapping.ts +50 -0
- package/resources/record-locks.md +1407 -0
- package/resources/recordLock.ts +173 -7
- package/resources/recordLockCoordinator.ts +3082 -0
- package/resources/replayLogs.ts +5 -0
- package/resources/replicatedApplyFailure.ts +77 -0
- package/resources/scheduler/DESIGN.md +40 -0
- package/resources/scheduler/scheduler.ts +4 -4
- package/resources/search.ts +169 -49
- package/resources/tracked.ts +73 -22
- package/resources/transactionBroadcast.ts +4 -4
- package/security/DESIGN.md +175 -0
- package/security/auth.ts +53 -24
- package/security/deferredAuthentication.ts +24 -2
- package/security/jsLoader.ts +9 -4
- package/server/DESIGN.md +275 -0
- package/server/REST.ts +42 -6
- package/server/http.ts +34 -2
- package/server/mqtt.ts +6 -4
- package/server/serverHelpers/Headers.ts +5 -1
- package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
- package/server/serverHelpers/Request.ts +33 -131
- package/server/serverHelpers/contentTypes.ts +188 -15
- package/server/serverHelpers/serverUtilities.ts +157 -27
- package/server/serverHelpers/uwsServer.ts +4 -1
- package/server/serverHelpers/webSocketCloseReason.ts +25 -0
- package/server/storageReclamation.ts +2 -2
- package/server/threads/isolatedApplications.ts +157 -0
- package/server/threads/logRotationTransport.ts +40 -0
- package/server/threads/manageThreads.js +254 -12
- package/server/threads/socketRouter.ts +217 -11
- package/server/threads/threadServer.js +30 -7
- package/studio/web/assets/{Chat-BnCBegQz.js → Chat-DADFFGe_.js} +1 -1
- package/studio/web/assets/{FloatingChat-CoDW1ySS.js → FloatingChat-D_mI-rZ7.js} +3 -3
- package/studio/web/assets/{apiToken-Bwk5BLXW.js → apiToken-c2NiSDHa.js} +1 -1
- package/studio/web/assets/{applications-DHxGi7JH.js → applications-DktUqh7G.js} +1 -1
- package/studio/web/assets/{cssMode-s0cWI-_M.js → cssMode-Cs_75Xhw.js} +1 -1
- package/studio/web/assets/{editor-DNcRHK54.js → editor-19b-Y1IN.js} +1 -1
- package/studio/web/assets/{html-Bdssedlg.js → html-DiYEQMpB.js} +1 -1
- package/studio/web/assets/{htmlMode-CoDlJ3fw.js → htmlMode-CmR0y7P_.js} +1 -1
- package/studio/web/assets/{index-D6sxmFLR.js → index-Dm0rfkJ7.js} +5 -5
- package/studio/web/assets/{index.lazy-tmU5BS8s.js → index.lazy-7vqt2CC3.js} +1 -1
- package/studio/web/assets/{javascript-B8meVSTH.js → javascript-BWtCFuOt.js} +1 -1
- package/studio/web/assets/{jsonMode-DpIPd35T.js → jsonMode-Buzzbv9y.js} +1 -1
- package/studio/web/assets/{languageServices-C_5FMJzQ.js → languageServices-SqsFWfTM.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-BIzNBzPK.js → lspLanguageFeatures-EMV5cmjo.js} +1 -1
- package/studio/web/assets/{notifications-CvZivSbh.js → notifications-CAB-LZWT.js} +1 -1
- package/studio/web/assets/{notifications-CQf18QKb.js → notifications-DRzmSRxM.js} +1 -1
- package/studio/web/assets/{profile-DdOwtntb.js → profile-BNKAl79n.js} +1 -1
- package/studio/web/assets/{regions-n69fwagr.js → regions-CUow_Zw2.js} +1 -1
- package/studio/web/assets/{register-PfWTCXWB.js → register-Dkt3WUMp.js} +2 -2
- package/studio/web/assets/{setComponentFile-Bg6O7X0S.js → setComponentFile-BZRfMD0N.js} +1 -1
- package/studio/web/assets/{setup-CUx_aUDl.js → setup-D_yiEPO2.js} +2 -2
- package/studio/web/assets/{status-D7BVKqX9.js → status-DhHh1Ge-.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-DBSyXzMr.js → toggleHighContrast-D7L1PDtV.js} +1 -1
- package/studio/web/assets/{tsMode-BByKCjBS.js → tsMode-CCwLk1YS.js} +1 -1
- package/studio/web/assets/{typescript-DDLnLpw9.js → typescript-BP1j1mjn.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-31CHGaHk.js → useEntityRestURL-D7bnYxLw.js} +1 -1
- package/studio/web/assets/{workers-pR3jRY9D.js → workers-tOuCNT17.js} +1 -1
- package/studio/web/assets/{xml-2iRnMhQO.js → xml-BSG_3mQT.js} +1 -1
- package/studio/web/assets/{yaml-Bf92gJpd.js → yaml-DxiLprBB.js} +1 -1
- package/studio/web/index.html +1 -1
- package/utility/DESIGN.md +55 -0
- package/utility/errors/hdbError.ts +54 -0
- package/utility/hdbTerms.ts +2 -0
- package/utility/logging/harper_logger.ts +210 -30
- package/utility/logging/logGenerationCoordinator.ts +196 -0
- package/utility/logging/logRotation.ts +367 -0
- package/utility/logging/logRotator.ts +196 -91
- package/utility/npmUtilities.ts +6 -4
- package/utility/watcherFallback.ts +0 -122
- package/validation/configValidator.ts +6 -3
|
@@ -0,0 +1,3082 @@
|
|
|
1
|
+
import { performance } from 'node:perf_hooks';
|
|
2
|
+
import { Packr } from 'msgpackr';
|
|
3
|
+
import harperLogger from '../utility/logging/harper_logger.ts';
|
|
4
|
+
import { ClientError, LockUnavailableError } from '../utility/errors/hdbError.ts';
|
|
5
|
+
import { MAX_LOCK_LEASE_MS, MIN_LOCK_LEASE_MS } from './recordLock.ts';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Cluster-wide record locks (harper#483, Phase 1): amortized per-record ownership.
|
|
9
|
+
* Design note: `docs/record-lock-ownership.md`.
|
|
10
|
+
*
|
|
11
|
+
* `Table.lock()` acquires the node's rocksdb-js key lock first, which bounds this process to one
|
|
12
|
+
* outstanding acquisition per key; only then does it run the cluster step here. That step has three
|
|
13
|
+
* levels at three very different rates:
|
|
14
|
+
*
|
|
15
|
+
* - **The home map** — `(generation, homes[])`, published by an operator through harper-pro and
|
|
16
|
+
* handed to core through `transport.homeMap()`. It is immutable for the life of its generation:
|
|
17
|
+
* core never computes it, never advances it, and never proceeds without it.
|
|
18
|
+
* - **The home node** — within a generation, a key's arbiter is a rendezvous hash over `homes[]`
|
|
19
|
+
* (§4.4). One arbiter per key is trivially exclusive, so there is no grant state machine at all:
|
|
20
|
+
* no deferral queues, no `(tsR, nodeId)` tiebreak, no synthesized grants, no split votes, no
|
|
21
|
+
* revocation protocol between peers.
|
|
22
|
+
* - **The delegation** — the exclusive right to admit critical sections on one key for a bounded
|
|
23
|
+
* time. While one is live, `lock()`/`unlock()` are pure Phase 0: the local key lock and **zero
|
|
24
|
+
* cluster messages**. Releasing the application lock does not release the delegation, so a node
|
|
25
|
+
* writing the same record repeatedly pays one round and then nothing.
|
|
26
|
+
*
|
|
27
|
+
* Two properties carry the safety argument, and both are enforced rather than assumed:
|
|
28
|
+
*
|
|
29
|
+
* - **One delegate per key per home.** A home never has two live delegations for a key, and a
|
|
30
|
+
* successor delegation is issued only after the predecessor's has been recalled-and-drained or has
|
|
31
|
+
* provably expired on the home's own clock plus skew. Every expiry decision on both sides is made
|
|
32
|
+
* on that side's monotonic clock; no remote timestamp is ever compared against a local one.
|
|
33
|
+
* - **A delegation bounds every handle it admitted.** An admission may not outlive its delegation, so
|
|
34
|
+
* recall revokes capability rather than merely closing the door (§6): the commit-time lease fence
|
|
35
|
+
* in `DatabaseTransaction` rejects a staged write whose handle has expired, and a recall expires
|
|
36
|
+
* those handles before the release is acknowledged.
|
|
37
|
+
*
|
|
38
|
+
* `nodeId` here is the globally stable node NAME. It is deliberately not the audit entry's `nodeId`:
|
|
39
|
+
* `nodeIdMapping.ts` hands out per-node short ids (0 is always local), so the same node has different
|
|
40
|
+
* ids on different nodes and any ordering built on them would order the same pair differently on two
|
|
41
|
+
* nodes.
|
|
42
|
+
*
|
|
43
|
+
* Successor freshness follows §7: a clean release carries inherited origin-log dependencies, the
|
|
44
|
+
* home advances the releasing origin to that entry's own position, and the next delegate cannot
|
|
45
|
+
* admit until its transport has made the set visible. Missing lineage takes the weaker recovery
|
|
46
|
+
* barrier and fails closed if that barrier cannot be established.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* How long a home waits before re-sending a recall that FAILED. A recall the delegate confirmed is
|
|
51
|
+
* never re-sent: it has stopped admitting, and the grant is then cleared by its release or by its own
|
|
52
|
+
* deadline. Without both rules a contender polling at 25 ms re-armed the recall on every pass and
|
|
53
|
+
* turned one handoff into an RPC storm lasting the rest of the delegation.
|
|
54
|
+
*/
|
|
55
|
+
export const RECALL_RETRY_MS = 1_000;
|
|
56
|
+
|
|
57
|
+
/** Margin a home adds to a delegation it issued, so the delegate always stops admitting first. */
|
|
58
|
+
export const LOCK_LEASE_SKEW_MS = 5_000;
|
|
59
|
+
/**
|
|
60
|
+
* How long a delegation runs, independent of any one caller's lock lease. It MUST be longer than the
|
|
61
|
+
* longest lease it will admit, or the amortization does not exist: a delegation sized to the caller's
|
|
62
|
+
* lease has no room left for the next lock, so every repeat `lock()` renews and pays a round trip —
|
|
63
|
+
* exactly the cost this design is built to remove.
|
|
64
|
+
*/
|
|
65
|
+
export const DELEGATION_LEASE_MS = MAX_LOCK_LEASE_MS + 60_000;
|
|
66
|
+
/**
|
|
67
|
+
* Below this an admission map is too small for its dead entries to matter, so the expiry sweep in
|
|
68
|
+
* `#pruneAdmissions` does not run at all; above it the map may reach twice the live set first.
|
|
69
|
+
*/
|
|
70
|
+
const ADMISSION_SWEEP_FLOOR = 64;
|
|
71
|
+
const TICK_INTERVAL_MS = 100;
|
|
72
|
+
/**
|
|
73
|
+
* How often `tick()` asks the transport whether this thread still coordinates. Far coarser than the
|
|
74
|
+
* tick because the answer only has to be sampled faster than a gap can matter, and it cannot matter
|
|
75
|
+
* below a full delegation lease: whatever coordinated during the gap is inside its own quarantine
|
|
76
|
+
* until then. At the tick rate an idle coordinating table would call the transport 10 times a second
|
|
77
|
+
* for the life of the process.
|
|
78
|
+
*/
|
|
79
|
+
const OWNERSHIP_POLL_MS = 1_000;
|
|
80
|
+
const WARN_INTERVAL_MS = 60_000;
|
|
81
|
+
/**
|
|
82
|
+
* Bounds the grants ONE TABLE's coordinator can accumulate — a home may never forget a delegation
|
|
83
|
+
* before its expiry, so the only bound available is a refusal to issue more. There is one coordinator
|
|
84
|
+
* per table, so the process-wide exposure is this times the number of tables under lock pressure; a
|
|
85
|
+
* true process-level bound is harper#2581 and is sized with the enablement measurements.
|
|
86
|
+
*/
|
|
87
|
+
const MAX_DELEGATIONS_PER_TABLE = 10_000;
|
|
88
|
+
/** Bounds what a single peer can make one table's home retain, so one node cannot exhaust it. */
|
|
89
|
+
const MAX_DELEGATIONS_PER_REQUESTER = 2_000;
|
|
90
|
+
/** Bounds expiry work per tick, so a burst of expiries cannot stall the event loop. */
|
|
91
|
+
const MAX_EXPIRIES_PER_TICK = 256;
|
|
92
|
+
/** How many entries a relay sweep may LOOK at per tick, so a thread holding many live relayed
|
|
93
|
+
* admissions does not walk all of them every 100 ms to find nothing. Well above any realistic count of
|
|
94
|
+
* concurrent off-owner locks on one thread, so an ordinary sweep still completes a full pass. */
|
|
95
|
+
const MAX_EXAMINED_PER_TICK = 4_096;
|
|
96
|
+
/** How long past its lease a relayed admission entry is kept before pruning (harper-pro#852), so the
|
|
97
|
+
* handle's own lease timer fires first and forwards its release to the owner rather than racing the
|
|
98
|
+
* sweep that would drop the entry it needs. A few ticks is plenty. */
|
|
99
|
+
const REMOTE_PRUNE_GRACE_MS = 500;
|
|
100
|
+
/**
|
|
101
|
+
* What an owner-worker acquire (harper-pro#852) reserves out of the caller's `waitMs` for the round
|
|
102
|
+
* trip, so the transport is asked for a wait it can answer WITHIN the caller's budget. `lock()` holds
|
|
103
|
+
* the native key for the whole wait, so the hop must come out of that budget, never on top of it:
|
|
104
|
+
* overshooting blocks every other worker on the key for the overshoot. Halved for a caller whose wait
|
|
105
|
+
* is shorter than the allowance, so a short wait still leaves the owner something to work with.
|
|
106
|
+
*
|
|
107
|
+
* Small on purpose. This covers only the core-to-transport boundary: a transport that relays bounds
|
|
108
|
+
* itself inside the wait it is handed and reserves its own margin for the hops it makes, so reserving
|
|
109
|
+
* a hop-sized allowance here as well would subtract the same round trip from the caller twice.
|
|
110
|
+
*/
|
|
111
|
+
const REMOTE_ACQUIRE_HOP_MS = 500;
|
|
112
|
+
/** Core's last-resort net on an owner-worker acquire, on top of the caller's `waitMs`. The transport
|
|
113
|
+
* already bounds itself inside that budget, so this fires only when it never returns at all — small,
|
|
114
|
+
* because it is a wedged-transport net and not a second full wait. */
|
|
115
|
+
const REMOTE_ACQUIRE_BACKSTOP_MS = 1_000;
|
|
116
|
+
const MAX_NODE_NAME_LENGTH = 255;
|
|
117
|
+
const MAX_LOCK_DEPENDENCIES = 1_024;
|
|
118
|
+
const MAX_DEPENDENCY_SETS_PER_TABLE = 20_000;
|
|
119
|
+
/**
|
|
120
|
+
* A 256 KiB filter keeps the false-positive rate near 1% through 200,000 distinct delegated keys
|
|
121
|
+
* per lock-active table and generation. Saturation remains safe — it selects recovery — but making
|
|
122
|
+
* that the normal path would turn every first lock on a new key into a cluster-wide barrier.
|
|
123
|
+
*/
|
|
124
|
+
const DELEGATED_KEY_FILTER_WORDS = 65_536;
|
|
125
|
+
/** A node whose identity resolved to one of these is not distinctive enough to be a ring member. */
|
|
126
|
+
const NON_DISTINCTIVE_NODE_NAMES = new Set(['127.0.0.1', '::1', 'localhost', '0.0.0.0']);
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The two control entries. `lockRequest`/`lockGrant` belonged to the Ricart–Agrawala rule the
|
|
130
|
+
* design note replaces; they never shipped enabled, so their nibbles were retired rather than
|
|
131
|
+
* migrated (`auditStore.ts`). Delegation request/grant/recall are unicast over the transport, not
|
|
132
|
+
* entries — the release stays on the replicated log because it is what orders a handoff behind the
|
|
133
|
+
* delegate's own data writes, and the barrier is on it because being replicated is its whole
|
|
134
|
+
* purpose (§7.2): a member commits one on request, and its position is the point a peer must have
|
|
135
|
+
* applied that origin through before a recovery-mode successor may admit.
|
|
136
|
+
*/
|
|
137
|
+
export type LockControlType = 'lockRelease' | 'lockBarrier';
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The operator-agreed map a key's home is derived from. Supplied by harper-pro; core never computes
|
|
141
|
+
* it and never advances it. Immutable for the life of a generation — nothing a node observes changes
|
|
142
|
+
* it, which is why no agreement protocol runs here (§4).
|
|
143
|
+
*/
|
|
144
|
+
export interface LockHomeMap {
|
|
145
|
+
/** Monotonic per database. Part of the fencing token, so it must never go backwards. */
|
|
146
|
+
generation: number;
|
|
147
|
+
/**
|
|
148
|
+
* Every node that participates in cluster record locks for this database — not only the ones an
|
|
149
|
+
* operator thinks of as arbiters. Order is irrelevant; the ring hashes each name independently.
|
|
150
|
+
*
|
|
151
|
+
* It is one set and not two because a home refuses a delegation to any node this list does not
|
|
152
|
+
* name (that is what keeps a decommissioned node from taking one), so a node absent from it can
|
|
153
|
+
* neither home a key nor lock one. Rendezvous hashing then makes every listed node the arbiter for
|
|
154
|
+
* its share of the ring, which is the property that costs a second list nothing.
|
|
155
|
+
*/
|
|
156
|
+
homes: string[];
|
|
157
|
+
/**
|
|
158
|
+
* This node's durably persisted, monotonic incarnation counter as a home (§5.1). A random value
|
|
159
|
+
* makes a stale reply identifiable but not ORDERABLE: a home that restarts and re-issues counter 1
|
|
160
|
+
* after having issued counter 50 would let a delayed counter-50 write defeat its successor.
|
|
161
|
+
*
|
|
162
|
+
* **Per coordination incarnation, not per process.** Coordinator state — including the delegation
|
|
163
|
+
* counter — is per-thread, so a replacement coordinating worker starts counting from zero. If the
|
|
164
|
+
* incarnation did not advance with it, the new worker would re-mint tokens its predecessor already
|
|
165
|
+
* issued, and §4.3's incarnation-bound quiescence acknowledgements would survive a restart that
|
|
166
|
+
* discarded everything they attested to.
|
|
167
|
+
*/
|
|
168
|
+
homeIncarnation: number;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* A delegation's fencing token, ordered lexicographically as
|
|
173
|
+
* `(generation, homeIncarnation, counter)`. Comparable across homes only within a generation, which
|
|
174
|
+
* is all that is needed: a key has exactly one home per generation.
|
|
175
|
+
*/
|
|
176
|
+
export type FencingToken = readonly [generation: number, homeIncarnation: number, counter: number];
|
|
177
|
+
export type LockDependency = readonly [origin: string, position: number];
|
|
178
|
+
export type LockDependencySet = readonly LockDependency[];
|
|
179
|
+
|
|
180
|
+
export function compareTokens(a: FencingToken, b: FencingToken): number {
|
|
181
|
+
return a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function isFencingToken(value: unknown): value is FencingToken {
|
|
185
|
+
return (
|
|
186
|
+
Array.isArray(value) &&
|
|
187
|
+
value.length === 3 &&
|
|
188
|
+
value.every((part) => typeof part === 'number' && Number.isFinite(part))
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export interface LockReleaseEntry {
|
|
193
|
+
type: 'lockRelease';
|
|
194
|
+
/** The locked record's id. Control entries carry it here, never as the audit entry's recordId. */
|
|
195
|
+
key: any;
|
|
196
|
+
/** Node that held the delegation being released. */
|
|
197
|
+
requester: string;
|
|
198
|
+
/**
|
|
199
|
+
* The released delegation's whole fencing token. The counter alone is NOT enough to identify it: a
|
|
200
|
+
* home that restarts, or whose coordinator is recreated, begins counting again, so a delayed
|
|
201
|
+
* release from a previous incarnation would match a live grant's counter and clear it while its
|
|
202
|
+
* delegate is still admitting.
|
|
203
|
+
*/
|
|
204
|
+
token: FencingToken;
|
|
205
|
+
/**
|
|
206
|
+
* Inherited clean-handoff lineage. Null hands back an unused first grant without advancing it; a
|
|
207
|
+
* renewed grant still drops retained lineage because its earlier token may have admitted writes.
|
|
208
|
+
* Absent means legacy/unknown.
|
|
209
|
+
*/
|
|
210
|
+
dependencies?: LockDependencySet | null;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The §7.2 recovery fence. It names no key and no token: it is appended after every transaction the
|
|
215
|
+
* writing node had committed when it was requested, so a peer that has applied that origin's log
|
|
216
|
+
* through this entry has applied all of them. The coordinator never acts on one.
|
|
217
|
+
*/
|
|
218
|
+
export interface LockBarrierEntry {
|
|
219
|
+
type: 'lockBarrier';
|
|
220
|
+
/** Supplied by the requesting transport; distinguishes barriers an origin stamped identically across a restart. */
|
|
221
|
+
nonce: number;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
export type LockControlEntry = LockReleaseEntry | LockBarrierEntry;
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* What a home replies to a delegation request. One shape rather than a discriminated union: the
|
|
228
|
+
* repo compiles with `strict: false`, where TypeScript does not narrow a union on a boolean literal,
|
|
229
|
+
* so a union here would type-check the denial fields as absent on every branch and then not enforce
|
|
230
|
+
* it. `granted` says which half is populated.
|
|
231
|
+
*/
|
|
232
|
+
export interface DelegationReply {
|
|
233
|
+
granted: boolean;
|
|
234
|
+
/** Granted only. */
|
|
235
|
+
token?: FencingToken;
|
|
236
|
+
/** Granted only. How long the delegate may admit for, as a DURATION — never a remote clock reading. */
|
|
237
|
+
leaseMs?: number;
|
|
238
|
+
/** Granted only. Null selects the recovery barrier; an array is the exact clean-handoff fence. */
|
|
239
|
+
dependencies?: LockDependencySet | null;
|
|
240
|
+
/**
|
|
241
|
+
* Denied only. `contended` is the one reason that means another node holds the key, and so the only
|
|
242
|
+
* one an exhausted wait may report as 423. `generation` (the two sides hold different home maps),
|
|
243
|
+
* `unknown-node` (this node is not named in the map) and `quarantine` (the home is inside its §4.3
|
|
244
|
+
* restart interval) all describe something other than contention, so each ends as a retryable 503
|
|
245
|
+
* rather than telling the caller a key nobody holds is held. `timeout` is not a home answer at all
|
|
246
|
+
* — it is this node's own deadline ending its probe — and never classifies a wait (DESIGN.md).
|
|
247
|
+
*/
|
|
248
|
+
reason?: 'contended' | 'generation' | 'unknown-node' | 'capacity' | 'not-home' | 'quarantine' | 'timeout';
|
|
249
|
+
/** Denied with `generation`, so a stale requester can re-derive the ring without another round trip. */
|
|
250
|
+
generation?: number;
|
|
251
|
+
retryAfterMs?: number;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** A completed home reply, bound to the route that produced it so a ring change retires it. */
|
|
255
|
+
interface LastCompletedReply {
|
|
256
|
+
reply: DelegationReply;
|
|
257
|
+
home: string;
|
|
258
|
+
generation: number;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export interface DelegationRequest {
|
|
262
|
+
key: any;
|
|
263
|
+
/** The asking node. Established by the transport, never read from an untrusted payload. */
|
|
264
|
+
requester: string;
|
|
265
|
+
generation: number;
|
|
266
|
+
leaseMs: number;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export interface DelegationRecall {
|
|
270
|
+
key: any;
|
|
271
|
+
token: FencingToken;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Supplied by harper-pro. Core never computes cluster topology; it only refuses to promise a
|
|
276
|
+
* cluster-wide lock that this contract cannot back.
|
|
277
|
+
*/
|
|
278
|
+
export interface ClusterLockTransport {
|
|
279
|
+
/**
|
|
280
|
+
* The operator-agreed home map for the database, or undefined while none is available — before the
|
|
281
|
+
* node has the current generation, or while peers disagree about its digest. Core fails closed on
|
|
282
|
+
* undefined rather than guessing a ring.
|
|
283
|
+
*
|
|
284
|
+
* **One obligation core cannot check, and relies on** (§4.3): a generation change is
|
|
285
|
+
* operator-sequenced, so `g+1` may only be answered here once the control plane's one-shot
|
|
286
|
+
* activation record is active — every old home quiesced (each acknowledgement bound to the
|
|
287
|
+
* acknowledger's `homeIncarnation`, so a restart during the drain invalidates it) or externally
|
|
288
|
+
* fenced, then `DELEGATION_LEASE_MS + skew` elapsed. Core cannot observe what happened on other
|
|
289
|
+
* nodes; the operator can.
|
|
290
|
+
*
|
|
291
|
+
* The other interval — a restart of *this* process — is core's own, because a generation does not
|
|
292
|
+
* advance on a restart and there is no external event to hang it on. See `#grantableAfterMono`.
|
|
293
|
+
*/
|
|
294
|
+
homeMap(database: string): LockHomeMap | undefined;
|
|
295
|
+
/**
|
|
296
|
+
* Overrides core's cold-start grant quarantine (§4.3). Set it only where a previous incarnation
|
|
297
|
+
* of this process provably issued nothing — a fresh database, a first start, or a test. Because the
|
|
298
|
+
* same attestation enables the virgin-key freshness fast path, it must prove that no earlier
|
|
299
|
+
* incarnation delegated any key under the current home-map generation at any time, not merely that
|
|
300
|
+
* its last delegation lease has elapsed. Omitted means core enforces the full
|
|
301
|
+
* `DELEGATION_LEASE_MS + skew` from that coordinator's construction — see `#grantableAfterMono` for
|
|
302
|
+
* why neither process start nor thread start is a sound anchor.
|
|
303
|
+
*/
|
|
304
|
+
grantableAfterMono?: number;
|
|
305
|
+
/**
|
|
306
|
+
* Whether this worker thread owns lock coordination for the process. Coordinator state is
|
|
307
|
+
* per-thread while the key lock it arbitrates is process-wide, so a second thread running its own
|
|
308
|
+
* delegations would arbitrate against a different view. Core fails closed off the owner thread.
|
|
309
|
+
*/
|
|
310
|
+
ownsCoordination(): boolean;
|
|
311
|
+
/** Ask `node` (the key's home) for a delegation. Unicast; rejects if the node is unreachable. */
|
|
312
|
+
requestDelegation(
|
|
313
|
+
node: string,
|
|
314
|
+
database: string,
|
|
315
|
+
table: string,
|
|
316
|
+
request: DelegationRequest
|
|
317
|
+
): Promise<DelegationReply>;
|
|
318
|
+
/** Home → delegate. Resolves once the delegate has drained and stopped admitting. */
|
|
319
|
+
recallDelegation(node: string, database: string, table: string, recall: DelegationRecall): Promise<void>;
|
|
320
|
+
/**
|
|
321
|
+
* Obtain an admission from the worker thread that coordinates this database, for a `lock()` served
|
|
322
|
+
* on a thread that is not the owner. Present only when the transport can relay across threads
|
|
323
|
+
* (harper-pro#852); a transport without it makes `acquire()` fail closed off the owner thread, as
|
|
324
|
+
* before. The returned `LockRound` was minted by the owner's coordinator; the caller's coordinator
|
|
325
|
+
* records it as a REMOTE admission and installs the handle's revoker locally, so a recall on the
|
|
326
|
+
* owner fences a write this thread's handle staged. `mintedMono` is comparable across threads
|
|
327
|
+
* because `performance.now()` shares one time origin process-wide.
|
|
328
|
+
*/
|
|
329
|
+
acquireOnOwner?(database: string, table: string, key: any, leaseMs: number, waitMs: number): Promise<LockRound>;
|
|
330
|
+
/** Release a remote admission on the owner thread (the counterpart of `acquireOnOwner`). */
|
|
331
|
+
releaseOnOwner?(database: string, table: string, key: any, admissionId: number): Promise<void> | void;
|
|
332
|
+
/**
|
|
333
|
+
* Establish an exact clean-handoff dependency set, or recover the strongest reachable-member
|
|
334
|
+
* barrier when `dependencies` is null. Recovery returns the captured positions for current lock
|
|
335
|
+
* participants that were made visible; it may drain additional replication peers without carrying
|
|
336
|
+
* them in the key's lineage. Clean waits may return void. Concurrent recovery snapshots should be
|
|
337
|
+
* coalesced.
|
|
338
|
+
*
|
|
339
|
+
* Recovery asks each reachable member for a `lockBarrier` entry (`writeLockBarrier`) and drains
|
|
340
|
+
* that member's stream through the position it returns. Core races this promise against the
|
|
341
|
+
* lock's own deadline but cannot cancel it; `deadlineMs` is the wait remaining at the call, so the
|
|
342
|
+
* transport can bound its own work to it instead of outliving the lock that asked.
|
|
343
|
+
*/
|
|
344
|
+
establishLockFreshness(
|
|
345
|
+
database: string,
|
|
346
|
+
table: string,
|
|
347
|
+
key: any,
|
|
348
|
+
dependencies: LockDependencySet | null,
|
|
349
|
+
deadlineMs: number
|
|
350
|
+
): Promise<LockDependencySet | void>;
|
|
351
|
+
/** Emit a control entry and return the committed entry's local origin-log position when available. */
|
|
352
|
+
writeControl?(table: string, entry: LockControlEntry): Promise<number | void> | number | void;
|
|
353
|
+
/**
|
|
354
|
+
* Assigned at registration so a transport can push a received entry in directly. `author` and
|
|
355
|
+
* `position` come from the authenticated origin-log header, never from the payload.
|
|
356
|
+
*/
|
|
357
|
+
onControlEntry?(database: string, table: string, entry: LockControlEntry, author: string, position: number): void;
|
|
358
|
+
/** Assigned at registration. Inbound delegation request from a peer, for a key this node homes. */
|
|
359
|
+
onDelegationRequest?(database: string, table: string, request: DelegationRequest): Promise<DelegationReply>;
|
|
360
|
+
/** Assigned at registration. Inbound recall from a key's home. */
|
|
361
|
+
onDelegationRecall?(database: string, table: string, recall: DelegationRecall): Promise<void>;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// A private structure dictionary, so a control payload can never contribute to — or depend on — the
|
|
365
|
+
// table's own, and never passes through schema projection (see writeLockControlEntry in Table.ts).
|
|
366
|
+
// Record mode stays ON deliberately: the reader is the receiving table's decoder (via
|
|
367
|
+
// `auditRecord.getValue`), which repurposes a range of positive fixints as structure ids. A payload
|
|
368
|
+
// packed without record mode writes an integer record key of 64..127 as a bare fixint, which that
|
|
369
|
+
// decoder then reads as a structure header and the whole entry fails to decode.
|
|
370
|
+
let controlStructures: unknown[] = [];
|
|
371
|
+
let controlPackr = new Packr({ structures: controlStructures });
|
|
372
|
+
|
|
373
|
+
export function encodeLockControlPayload(entry: LockControlEntry): Uint8Array {
|
|
374
|
+
if (entry.type === 'lockBarrier') return controlPackr.pack([1, entry.nonce]);
|
|
375
|
+
const [generation, homeIncarnation, counter] = entry.token;
|
|
376
|
+
if (entry.dependencies === undefined)
|
|
377
|
+
return controlPackr.pack([entry.key, entry.requester, generation, homeIncarnation, counter]);
|
|
378
|
+
return controlPackr.pack([1, entry.key, entry.requester, generation, homeIncarnation, counter, entry.dependencies]);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function isNodeName(value: unknown): value is string {
|
|
382
|
+
return typeof value === 'string' && value.length > 0 && value.length <= MAX_NODE_NAME_LENGTH;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function isDuration(value: unknown, min: number, max: number): value is number {
|
|
386
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= min && value <= max;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* Accept exactly what `ordered-binary` encodes, because that is what `keyIdOf` runs on the key.
|
|
391
|
+
* Refusing a shape the encoder handles makes the home answer `not-home` for a key nobody holds, and
|
|
392
|
+
* the requester retries that to its own 423. A `Uint8Array` returns from the unpack as a `Buffer`
|
|
393
|
+
* and a small `bigint` as a `number`; both encode to the same stored key, which is the identity at
|
|
394
|
+
* issue rather than the JS value.
|
|
395
|
+
*/
|
|
396
|
+
function isEncodableKey(value: unknown): boolean {
|
|
397
|
+
const type = typeof value;
|
|
398
|
+
if (type === 'string' || type === 'bigint' || type === 'number' || type === 'boolean') return true;
|
|
399
|
+
if (value === null || value instanceof Uint8Array) return true;
|
|
400
|
+
return Array.isArray(value) && (value as unknown[]).every(isEncodableKey);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Decode a received control payload, or undefined when it is not one this version understands. The
|
|
405
|
+
* tuple length is validated exactly: a future version that grows the payload must bump the type
|
|
406
|
+
* rather than widen this one, since a partially-understood release would clear a delegation on terms
|
|
407
|
+
* the sender did not intend.
|
|
408
|
+
*/
|
|
409
|
+
export function decodeLockControlPayload(type: unknown, value: unknown): LockControlEntry | undefined {
|
|
410
|
+
if (type !== 'lockRelease' && type !== 'lockBarrier') return undefined;
|
|
411
|
+
let tuple: unknown;
|
|
412
|
+
try {
|
|
413
|
+
tuple = value instanceof Uint8Array ? controlPackr.unpack(value) : value;
|
|
414
|
+
} catch {
|
|
415
|
+
return undefined;
|
|
416
|
+
}
|
|
417
|
+
if (!Array.isArray(tuple)) return undefined;
|
|
418
|
+
if (type === 'lockBarrier') {
|
|
419
|
+
if (tuple.length !== 2 || tuple[0] !== 1) return undefined;
|
|
420
|
+
const nonce = tuple[1];
|
|
421
|
+
return typeof nonce === 'number' && Number.isFinite(nonce) ? { type: 'lockBarrier', nonce } : undefined;
|
|
422
|
+
}
|
|
423
|
+
if (tuple.length !== 5 && tuple.length !== 7) return undefined;
|
|
424
|
+
try {
|
|
425
|
+
return decodeTuple(tuple);
|
|
426
|
+
} catch {
|
|
427
|
+
// isEncodableKey recurses; a deeply nested array in a peer or replayed payload would otherwise
|
|
428
|
+
// raise a RangeError into the replicated apply loop instead of being dropped as malformed.
|
|
429
|
+
return undefined;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function decodeTuple(tuple: unknown[]): LockReleaseEntry | undefined {
|
|
434
|
+
const versioned = tuple.length === 7;
|
|
435
|
+
if (versioned && tuple[0] !== 1) return undefined;
|
|
436
|
+
const offset = versioned ? 1 : 0;
|
|
437
|
+
const [key, requester, generation, homeIncarnation, counter] = tuple.slice(offset, offset + 5);
|
|
438
|
+
if (!isEncodableKey(key) || !isNodeName(requester)) return undefined;
|
|
439
|
+
for (const part of [generation, homeIncarnation, counter])
|
|
440
|
+
if (typeof part !== 'number' || !Number.isFinite(part)) return undefined;
|
|
441
|
+
let dependencies: LockDependencySet | null | undefined;
|
|
442
|
+
if (versioned) {
|
|
443
|
+
const rawDependencies = tuple[6];
|
|
444
|
+
if (rawDependencies === null) dependencies = null;
|
|
445
|
+
else dependencies = normalizeDependencies(rawDependencies);
|
|
446
|
+
}
|
|
447
|
+
const entry: LockReleaseEntry = {
|
|
448
|
+
type: 'lockRelease',
|
|
449
|
+
key,
|
|
450
|
+
requester,
|
|
451
|
+
token: [generation, homeIncarnation, counter] as FencingToken,
|
|
452
|
+
};
|
|
453
|
+
if (versioned) entry.dependencies = dependencies;
|
|
454
|
+
return entry;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function normalizeDependencies(value: unknown, homes?: readonly string[]): LockDependencySet | undefined {
|
|
458
|
+
if (!Array.isArray(value) || value.length > MAX_LOCK_DEPENDENCIES) return undefined;
|
|
459
|
+
const positions = new Map<string, number>();
|
|
460
|
+
const allowedOrigins = homes && new Set(homes);
|
|
461
|
+
for (const dependency of value) {
|
|
462
|
+
if (!Array.isArray(dependency) || dependency.length !== 2) return undefined;
|
|
463
|
+
const [origin, position] = dependency;
|
|
464
|
+
if (!isNodeName(origin) || typeof position !== 'number' || !Number.isFinite(position) || position < 0)
|
|
465
|
+
return undefined;
|
|
466
|
+
if (allowedOrigins && !allowedOrigins.has(origin)) return undefined;
|
|
467
|
+
if (positions.has(origin)) return undefined;
|
|
468
|
+
positions.set(origin, position);
|
|
469
|
+
}
|
|
470
|
+
return [...positions].sort(([a], [b]) => a.localeCompare(b));
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
class DelegatedKeyFilter {
|
|
474
|
+
#bits = new Uint32Array(DELEGATED_KEY_FILTER_WORDS);
|
|
475
|
+
|
|
476
|
+
add(key: unknown): void {
|
|
477
|
+
const [first, second] = this.#hashes(key);
|
|
478
|
+
for (let index = 0; index < 4; index++) this.#set((first + Math.imul(index, second)) >>> 0);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
has(key: unknown): boolean {
|
|
482
|
+
const [first, second] = this.#hashes(key);
|
|
483
|
+
for (let index = 0; index < 4; index++) if (!this.#get((first + Math.imul(index, second)) >>> 0)) return false;
|
|
484
|
+
return true;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
clear(): void {
|
|
488
|
+
this.#bits.fill(0);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
copyFrom(other: DelegatedKeyFilter): void {
|
|
492
|
+
this.#bits.set(other.#bits);
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
#set(hash: number): void {
|
|
496
|
+
const bit = hash % (this.#bits.length * 32);
|
|
497
|
+
this.#bits[bit >>> 5] |= 1 << (bit & 31);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
#get(hash: number): boolean {
|
|
501
|
+
const bit = hash % (this.#bits.length * 32);
|
|
502
|
+
return (this.#bits[bit >>> 5] & (1 << (bit & 31))) !== 0;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
#hashes(key: unknown): [number, number] {
|
|
506
|
+
const value = `${typeof key}:${String(key)}`;
|
|
507
|
+
let first = 0x811c9dc5;
|
|
508
|
+
let second = 0x9e3779b9;
|
|
509
|
+
for (let index = 0; index < value.length; index++) {
|
|
510
|
+
first = Math.imul(first ^ value.charCodeAt(index), 0x01000193);
|
|
511
|
+
second = Math.imul(second ^ value.charCodeAt(index), 0x85ebca6b);
|
|
512
|
+
}
|
|
513
|
+
return [first >>> 0, (second | 1) >>> 0];
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Rendezvous (highest-random-weight) hash: a key's home is the node with the greatest score for that
|
|
519
|
+
* key. Chosen over a modulo of a key hash because a generation change moves only the keys homed on a
|
|
520
|
+
* departing node, rather than re-homing the whole space — which matters because every re-homed key
|
|
521
|
+
* pays the §7.2 recovery path on its next lock.
|
|
522
|
+
*
|
|
523
|
+
* The hash is FNV-1a over the member name and the key's stable id. It does not need to be
|
|
524
|
+
* cryptographic: it is not a defense against anything, only a deterministic agreement between nodes
|
|
525
|
+
* that already agree on `homes`.
|
|
526
|
+
*/
|
|
527
|
+
function scoreFor(member: string, keyId: string): number {
|
|
528
|
+
let hash = 0x811c9dc5;
|
|
529
|
+
for (let i = 0; i < member.length; i++) {
|
|
530
|
+
hash ^= member.charCodeAt(i);
|
|
531
|
+
hash = Math.imul(hash, 0x01000193);
|
|
532
|
+
}
|
|
533
|
+
// A separator that cannot appear in either operand, so ("ab","c") and ("a","bc") cannot collide.
|
|
534
|
+
hash ^= 0xff;
|
|
535
|
+
hash = Math.imul(hash, 0x01000193);
|
|
536
|
+
for (let i = 0; i < keyId.length; i++) {
|
|
537
|
+
hash ^= keyId.charCodeAt(i);
|
|
538
|
+
hash = Math.imul(hash, 0x01000193);
|
|
539
|
+
}
|
|
540
|
+
return hash >>> 0;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* The string the ring hashes for a key. §4.4 scopes it by database and table: hashing the record id
|
|
545
|
+
* alone would home id `42` in every table of every database on the same node, concentrating unrelated
|
|
546
|
+
* hot keys on one arbiter. The separator cannot appear in a name or a stringified key id.
|
|
547
|
+
*/
|
|
548
|
+
export function ringKeyFor(database: string, table: string, keyId: unknown): string {
|
|
549
|
+
return `${database}\u0000${table}\u0000${String(keyId)}`;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
export function homeFor(keyId: string, homes: string[]): string | undefined {
|
|
553
|
+
let best: string | undefined;
|
|
554
|
+
let bestScore = -1;
|
|
555
|
+
for (const home of homes) {
|
|
556
|
+
const score = scoreFor(home, keyId);
|
|
557
|
+
// Ties break on the name so every node picks the same home from the same set.
|
|
558
|
+
if (score > bestScore || (score === bestScore && best !== undefined && home > best)) {
|
|
559
|
+
bestScore = score;
|
|
560
|
+
best = home;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
return best;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/** A delegation this node holds: the right to admit critical sections on one key. */
|
|
567
|
+
interface Delegation {
|
|
568
|
+
key: any;
|
|
569
|
+
token: FencingToken;
|
|
570
|
+
dependencies: LockDependencySet;
|
|
571
|
+
/** Monotonic deadline on THIS node. A delegate stops admitting here. */
|
|
572
|
+
expiresMono: number;
|
|
573
|
+
/** Set by a recall. No new admission may start, but live ones are drained first. */
|
|
574
|
+
recalled: boolean;
|
|
575
|
+
/**
|
|
576
|
+
* Every admission this delegation is still answerable for, by id. An entry stays here after its
|
|
577
|
+
* caller unlocks: §6 revokes CAPABILITY, not admission. A caller that staged a write and then
|
|
578
|
+
* called `unlock()` has nothing left to wait on, but its write is still uncommitted and would land
|
|
579
|
+
* after the successor was admitted — so surrender has to fence it, which means keeping its revoker
|
|
580
|
+
* until its own lease runs out or authority is lost.
|
|
581
|
+
*
|
|
582
|
+
* Addressed by id through `#admissions` rather than owned by the delegation OBJECT, so a handle
|
|
583
|
+
* survives a renewal with its delegation and is revoked with it when authority is actually lost.
|
|
584
|
+
*/
|
|
585
|
+
admissions: Map<number, Admission>;
|
|
586
|
+
/**
|
|
587
|
+
* How many of those admissions have not unlocked yet. A drain waits for THIS to reach zero — an
|
|
588
|
+
* unlocked-but-staged write is revoked rather than waited for (harper#2580).
|
|
589
|
+
*/
|
|
590
|
+
holding: number;
|
|
591
|
+
/** `admissions.size` at which the next full expiry sweep runs; see `#pruneAdmissions`. */
|
|
592
|
+
sweepAtSize: number;
|
|
593
|
+
/** Resolvers waiting for the drain to finish, so recall can reply once rather than poll. */
|
|
594
|
+
drained?: (() => void)[];
|
|
595
|
+
/** The one in-flight `#surrender` for this delegation; see there for why it is memoized. */
|
|
596
|
+
surrendering?: Promise<void>;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* One `lock()` this thread admitted from the OWNER thread (harper-pro#852), addressed by a LOCAL id
|
|
601
|
+
* distinct from the owner's own admission id (see `#remoteByOwnerId`), so a stale local admission can
|
|
602
|
+
* never share a number with an incoming owner-minted one. `revoke` fences this thread's handle;
|
|
603
|
+
* `revoked` latches a revoke that landed before `registerAdmission` supplied the real revoker.
|
|
604
|
+
* `fenceWaiters` resolve once the handle is provably fenced — the owner waits on them before it writes
|
|
605
|
+
* the release, so an ack never claims a fence the handle has not actually taken.
|
|
606
|
+
*/
|
|
607
|
+
interface RemoteAdmission {
|
|
608
|
+
/** The owner's admission id, echoed on release and matched by an inbound revoke. */
|
|
609
|
+
ownerAdmissionId: number;
|
|
610
|
+
key: any;
|
|
611
|
+
/** The caller-side handle fence — today always `Table.lock`'s `() => handle.revokeLease()`, which is
|
|
612
|
+
* synchronous and total, never the owner-side async relay revoker (that lives on a delegation
|
|
613
|
+
* `Admission` instead). `registerAdmission` accepts an async revoker all the same, so the ack paths
|
|
614
|
+
* here settle on the outcome rather than assuming this one. */
|
|
615
|
+
revoke: () => void;
|
|
616
|
+
/** Monotonic deadline of the handle's own lease; the entry is dropped once past it. */
|
|
617
|
+
expiresMono: number;
|
|
618
|
+
revoked: boolean;
|
|
619
|
+
fenceWaiters: (() => void)[];
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/** One `lock()` admitted under a delegation. */
|
|
623
|
+
interface Admission {
|
|
624
|
+
/**
|
|
625
|
+
* Fences the handle's write capability. A no-op until `registerAdmission` supplies the real one. A
|
|
626
|
+
* handle admitted on another worker thread revokes over a message and resolves once that thread has
|
|
627
|
+
* fenced it; `#surrender` waits for that (or the admission's own lease) before writing the release.
|
|
628
|
+
*/
|
|
629
|
+
revoke: () => void | Promise<void>;
|
|
630
|
+
/** Monotonic deadline of the handle's OWN lease, after which it fences itself and can be dropped. */
|
|
631
|
+
expiresMono: number;
|
|
632
|
+
/** False once the caller unlocked: it no longer blocks a drain, but is still revocable. */
|
|
633
|
+
holding: boolean;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/** A delegation this node issued as a key's home. */
|
|
637
|
+
interface HomeGrant {
|
|
638
|
+
key: any;
|
|
639
|
+
delegate: string;
|
|
640
|
+
token: FencingToken;
|
|
641
|
+
/** Requirement handed to this delegate; reused if the same node renews after losing local state. */
|
|
642
|
+
dependencies: LockDependencySet | null;
|
|
643
|
+
/** This grant advanced from an earlier token that may already have admitted writes. */
|
|
644
|
+
renewed?: boolean;
|
|
645
|
+
/**
|
|
646
|
+
* Monotonic deadline on THIS node, set to the delegate's lease PLUS skew. The home always outwaits
|
|
647
|
+
* the delegate, so it can never re-grant a key the previous delegate still believes it holds.
|
|
648
|
+
*/
|
|
649
|
+
expiresMono: number;
|
|
650
|
+
recalling?: Promise<void>;
|
|
651
|
+
/** Set once the delegate confirmed it stopped admitting, so the recall is never re-sent. */
|
|
652
|
+
recallConfirmed?: boolean;
|
|
653
|
+
/** After a FAILED recall, the earliest this home may try that delegate again (`RECALL_RETRY_MS`). */
|
|
654
|
+
recallRetryAfterMono?: number;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
interface PendingDelegation {
|
|
658
|
+
key: any;
|
|
659
|
+
token: FencingToken;
|
|
660
|
+
recalled: boolean;
|
|
661
|
+
recalledPromise: Promise<void>;
|
|
662
|
+
markRecalled: () => void;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const PENDING_DELEGATION_RECALLED = Symbol('pending delegation recalled');
|
|
666
|
+
|
|
667
|
+
function createPendingDelegation(key: any, token: FencingToken, recalledBeforeReply: boolean): PendingDelegation {
|
|
668
|
+
let resolveRecall: () => void;
|
|
669
|
+
const pending: PendingDelegation = {
|
|
670
|
+
key,
|
|
671
|
+
token,
|
|
672
|
+
recalled: false,
|
|
673
|
+
recalledPromise: new Promise((resolve) => (resolveRecall = resolve)),
|
|
674
|
+
markRecalled() {
|
|
675
|
+
if (pending.recalled) return;
|
|
676
|
+
pending.recalled = true;
|
|
677
|
+
resolveRecall();
|
|
678
|
+
},
|
|
679
|
+
};
|
|
680
|
+
if (recalledBeforeReply) pending.markRecalled();
|
|
681
|
+
return pending;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
interface PendingRequest {
|
|
685
|
+
recalledToken?: FencingToken;
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
export interface LockRound {
|
|
689
|
+
/** The identity to stamp the holder's writes with. */
|
|
690
|
+
tsR: number;
|
|
691
|
+
/** The monotonic reading the admission's lease is measured from. */
|
|
692
|
+
mintedMono: number;
|
|
693
|
+
/**
|
|
694
|
+
* Identifies THIS admission for the life of the handle it produced. The caller hands it back on
|
|
695
|
+
* registration and on release. An id rather than the delegation's token, because the token changes
|
|
696
|
+
* on renewal while the admission does not: addressing by token made a renewed delegation lose
|
|
697
|
+
* track of handles it was still responsible for.
|
|
698
|
+
*/
|
|
699
|
+
admissionId: number;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
export interface LockCoordinatorOptions {
|
|
703
|
+
database: string;
|
|
704
|
+
table: string;
|
|
705
|
+
/** This node's globally stable name. */
|
|
706
|
+
nodeId: string;
|
|
707
|
+
transport: ClusterLockTransport;
|
|
708
|
+
/**
|
|
709
|
+
* Emit one control entry. Core passes the transport's own `writeControl` when it has one and its
|
|
710
|
+
* transaction-log writer otherwise, since writing to the local log IS the send.
|
|
711
|
+
*/
|
|
712
|
+
writeControl: (entry: LockControlEntry) => Promise<number | void> | number | void;
|
|
713
|
+
/** Stable map key for a record id; core passes `writeKeyId`. */
|
|
714
|
+
keyIdOf: (key: any) => unknown;
|
|
715
|
+
/** Mints the holder's stamp; core passes the primary store's monotonic timestamp. */
|
|
716
|
+
nextTimestamp: () => number;
|
|
717
|
+
/** Monotonic clock. Every expiry decision is made on this. */
|
|
718
|
+
monotonic?: () => number;
|
|
719
|
+
skewMs?: number;
|
|
720
|
+
/**
|
|
721
|
+
* The coordinator this one replaces, when a transport is re-registered (a component reload is
|
|
722
|
+
* enough). Its live authority is MOVED here rather than discarded: the transport changed, but the
|
|
723
|
+
* handles it admitted did not, and a successor that started with an empty grant table could hand
|
|
724
|
+
* the same key to another node with no lease time elapsed. That is the "a home may never forget a
|
|
725
|
+
* grant before its expiry" rule (§8) applied across the swap rather than only within one
|
|
726
|
+
* coordinator's life. §11 of the design note calls for exactly this — carry live authority across,
|
|
727
|
+
* or fence and settle every outstanding handle before granting; carrying it across costs nothing.
|
|
728
|
+
*/
|
|
729
|
+
adopt?: LockCoordinator;
|
|
730
|
+
/**
|
|
731
|
+
* Overrides the cold-start grant quarantine (§4.3). Pass `-Infinity` only where a previous
|
|
732
|
+
* incarnation provably issued nothing — a fresh database, or a test. Required when `monotonic` is
|
|
733
|
+
* an injected clock the test drives itself. See `#grantableAfterMono`.
|
|
734
|
+
*/
|
|
735
|
+
grantableAfterMono?: number;
|
|
736
|
+
/** False in tests, which drive `tick()` themselves. */
|
|
737
|
+
autoTick?: boolean;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
/**
|
|
741
|
+
* What a coordinator closed WITHOUT a successor leaves for its eventual replacement, per table.
|
|
742
|
+
*
|
|
743
|
+
* `close()` drops the grant table, but the delegations those grants authorize are still live on other
|
|
744
|
+
* nodes until their own deadlines — closing is a local event that no peer observes. A replacement
|
|
745
|
+
* built before then must not grant those keys again, and must not restart the counter into tokens the
|
|
746
|
+
* closed coordinator already issued. `#grantableAfterMono` covers the same hazard across a process
|
|
747
|
+
* RESTART; this covers it across an unregister and re-register inside one process, which a
|
|
748
|
+
* process-start reading cannot see.
|
|
749
|
+
*/
|
|
750
|
+
const retiredCoordinators = new Map<string, { grantableAfterMono: number; counter: number }>();
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* The highest home-map generation this thread has acted under, per DATABASE — the scope the generation
|
|
754
|
+
* itself has, not the per-table scope a token is compared at. A generation is the high-order component
|
|
755
|
+
* of every fencing token (§5.1), so going backwards re-mints tokens that order BELOW ones already
|
|
756
|
+
* handed out, and a delayed write under the newer generation then defeats its successor. Refusing it
|
|
757
|
+
* database-wide is strictly stronger than refusing it per table and costs nothing: one rolled-back
|
|
758
|
+
* publish is one event. The map is operator-published, so the rollback route is a configuration
|
|
759
|
+
* restore or a partial publish rather than a protocol bug — which is why it is refused here rather
|
|
760
|
+
* than assumed away. Remembering it across a restart, and across threads, is harper-pro's half.
|
|
761
|
+
*/
|
|
762
|
+
const highestGeneration = new Map<string, number>();
|
|
763
|
+
|
|
764
|
+
/** Bound one drain step so a single unresponsive delegate cannot consume the whole transition budget. */
|
|
765
|
+
function withDeadline<T>(work: Promise<T>, ms: number, message = 'the quiesce deadline elapsed'): Promise<T> {
|
|
766
|
+
if (!(ms > 0)) return Promise.reject(new Error(message));
|
|
767
|
+
const timer = delay(ms);
|
|
768
|
+
work.then(
|
|
769
|
+
() => timer.cancel(),
|
|
770
|
+
() => timer.cancel()
|
|
771
|
+
);
|
|
772
|
+
return Promise.race([
|
|
773
|
+
work,
|
|
774
|
+
timer.promise.then<T>(() => {
|
|
775
|
+
throw new Error(message);
|
|
776
|
+
}),
|
|
777
|
+
]);
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
const tickingCoordinators = new Set<LockCoordinator>();
|
|
781
|
+
/**
|
|
782
|
+
* Every coordinator alive on this thread, so a membership transition can quiesce a whole database
|
|
783
|
+
* rather than one table at a time (harper-pro#856). The per-`(database, table)` resolvers cannot
|
|
784
|
+
* enumerate: they answer a name you already have.
|
|
785
|
+
*/
|
|
786
|
+
const liveCoordinators = new Set<LockCoordinator>();
|
|
787
|
+
let tickTimer: ReturnType<typeof setInterval> | undefined;
|
|
788
|
+
function ensureTicking() {
|
|
789
|
+
if (tickTimer || tickingCoordinators.size === 0) return;
|
|
790
|
+
tickTimer = setInterval(() => {
|
|
791
|
+
for (const coordinator of tickingCoordinators) {
|
|
792
|
+
try {
|
|
793
|
+
coordinator.tick();
|
|
794
|
+
} catch (error) {
|
|
795
|
+
warnOnce('lock coordinator tick failed', error);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
if (tickingCoordinators.size === 0) {
|
|
799
|
+
clearInterval(tickTimer);
|
|
800
|
+
tickTimer = undefined;
|
|
801
|
+
}
|
|
802
|
+
}, TICK_INTERVAL_MS);
|
|
803
|
+
tickTimer.unref?.();
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
/** Stands in until `registerAdmission` supplies the handle's real revoker. */
|
|
807
|
+
function noRevoke() {}
|
|
808
|
+
|
|
809
|
+
/**
|
|
810
|
+
* Fire a revoker in a fire-and-forget context, absorbing both a synchronous throw and a rejected
|
|
811
|
+
* promise. A relayed revoker is `() => Promise<void>` and may reject on a dead sibling port; a
|
|
812
|
+
* discarded rejection would exit the worker under Node's default policy, taking every coordinator on
|
|
813
|
+
* the thread. Callers that must WAIT for the fence (`#revokeAllAndSettle`, `revokeRemoteAdmission`)
|
|
814
|
+
* handle the outcome themselves and do not use this.
|
|
815
|
+
*/
|
|
816
|
+
function isPromiseLike(value: unknown): value is Promise<unknown> {
|
|
817
|
+
return value != null && typeof (value as Promise<unknown>).then === 'function';
|
|
818
|
+
}
|
|
819
|
+
function fireRevokeAndForget(revoke: () => void | Promise<void>): void {
|
|
820
|
+
try {
|
|
821
|
+
const outcome = revoke();
|
|
822
|
+
if (isPromiseLike(outcome))
|
|
823
|
+
outcome.catch((error) => warnOnce('a fire-and-forget record lock revoke failed', error));
|
|
824
|
+
} catch (error) {
|
|
825
|
+
warnOnce('a fire-and-forget record lock revoke threw', error);
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Rate-limit rather than latch. These messages report a transport, writer or revoker that failed, and
|
|
831
|
+
* a latch for the life of the process would show an operator the first occurrence and then hide a
|
|
832
|
+
* fault that persists for days. One per message per window is enough to keep a hot loop from flooding
|
|
833
|
+
* the log while still showing that the condition is ongoing.
|
|
834
|
+
*/
|
|
835
|
+
const warnedMessages = new Map<string, number>();
|
|
836
|
+
function warnOnce(message: string, detail?: unknown) {
|
|
837
|
+
const now = performance.now();
|
|
838
|
+
const last = warnedMessages.get(message);
|
|
839
|
+
if (last !== undefined && now - last < WARN_INTERVAL_MS) return;
|
|
840
|
+
warnedMessages.set(message, now);
|
|
841
|
+
harperLogger.warn?.(message, detail);
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
export class LockCoordinator {
|
|
845
|
+
readonly database: string;
|
|
846
|
+
readonly table: string;
|
|
847
|
+
readonly nodeId: string;
|
|
848
|
+
readonly transport: ClusterLockTransport;
|
|
849
|
+
#writeControl: (entry: LockControlEntry) => Promise<number | void> | number | void;
|
|
850
|
+
#keyIdOf: (key: any) => unknown;
|
|
851
|
+
#nextTimestamp: () => number;
|
|
852
|
+
#monotonic: () => number;
|
|
853
|
+
#skewMs: number;
|
|
854
|
+
#autoTick: boolean;
|
|
855
|
+
/** Whoever this coordinator's authority was handed to, for replies that land after the swap. */
|
|
856
|
+
#successor: LockCoordinator | undefined;
|
|
857
|
+
/** Set when this coordinator's state was moved to a successor, so `close()` must not expire it. */
|
|
858
|
+
#handedOff = false;
|
|
859
|
+
/**
|
|
860
|
+
* Monotonic reading before which this coordinator may not grant as a home — the §4.3 restart
|
|
861
|
+
* quarantine, and core's own to enforce.
|
|
862
|
+
*
|
|
863
|
+
* A coordinator that started cold has no record of the delegations a previous incarnation issued,
|
|
864
|
+
* and those can still be admitting on their holders. Nothing external bounds them: the home map is
|
|
865
|
+
* immutable, so its generation does not advance merely because a process or a worker restarted. The
|
|
866
|
+
* only instant core can prove nothing else was granting under is this coordinator's own
|
|
867
|
+
* construction, so the quarantine runs `DELEGATION_LEASE_MS + skew` from there — by which point
|
|
868
|
+
* every delegation a previous incarnation could have issued has expired. It costs availability on
|
|
869
|
+
* this node's own share of the ring and nothing elsewhere: keys homed on other nodes are acquired
|
|
870
|
+
* immediately, and `adopt` plus the retirement record waive it wherever a predecessor's authority
|
|
871
|
+
* is actually known.
|
|
872
|
+
*
|
|
873
|
+
* A deployment that can prove a previous incarnation issued nothing overrides it through
|
|
874
|
+
* `ClusterLockTransport.grantableAfterMono`. Generation CHANGES need nothing here: the
|
|
875
|
+
* delegate-side generation check in `#liveDelegation` is what stops the old delegate.
|
|
876
|
+
*/
|
|
877
|
+
#grantableAfterMono: number;
|
|
878
|
+
/**
|
|
879
|
+
* How long until this coordinator can rule out authority issued before it took over — 0 when it
|
|
880
|
+
* already can. Non-mutating, unlike `#ownershipHorizon`, which records ownership as a side effect.
|
|
881
|
+
*
|
|
882
|
+
* The waiver is deliberately NOT consulted. `grantableAfterMono` attests that no previous
|
|
883
|
+
* INCARNATION OF THIS PROCESS delegated; it says nothing about a sibling thread that was
|
|
884
|
+
* coordinating until this instant, and a coordinator built while already owning keeps the waiver
|
|
885
|
+
* without ever observing that handoff — which let a takeover worker in a first-incarnation process
|
|
886
|
+
* report a clean drain while the previous owner's delegates were still admitting.
|
|
887
|
+
*
|
|
888
|
+
* The cost is that a freshly built coordinator cannot prove quiescence for a full lease. That falls
|
|
889
|
+
* only on the case that does not need the proof: a node with no delegations yet is bootstrapping
|
|
890
|
+
* generation 1, where there is nothing to drain and no interval to skip.
|
|
891
|
+
*/
|
|
892
|
+
unprovenOwnershipMs(): number {
|
|
893
|
+
const horizon = DELEGATION_LEASE_MS + this.#skewMs;
|
|
894
|
+
if (this.#ownedSinceMono === undefined) return horizon;
|
|
895
|
+
return Math.max(0, this.#ownedSinceMono + horizon - this.#monotonic());
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* When this coordinator was last observed to own coordination, and `undefined` while it does not.
|
|
899
|
+
*
|
|
900
|
+
* The quarantine has to run from here and not only from construction: a coordinator is built when a
|
|
901
|
+
* transport registers, but `ownsCoordination()` can flip to true long afterwards — a thread that
|
|
902
|
+
* took over from an owner that died. By then the construction horizon has aged out, and this
|
|
903
|
+
* coordinator would grant immediately over delegations the previous OWNER issued. Regaining
|
|
904
|
+
* ownership re-arms it for the same reason: something else was coordinating in between.
|
|
905
|
+
*/
|
|
906
|
+
#ownedSinceMono: number | undefined;
|
|
907
|
+
/** Rate-limits `tick()`'s ownership poll to `OWNERSHIP_POLL_MS`. */
|
|
908
|
+
#lastOwnershipPollMono = -Infinity;
|
|
909
|
+
/**
|
|
910
|
+
* The `homeIncarnation` this coordinator has been coordinating under. A different one is the
|
|
911
|
+
* transport saying another coordination incarnation ran for this node (§5.1), which is the only
|
|
912
|
+
* statement about CONTINUITY available here — see `#ownershipHorizon`.
|
|
913
|
+
*/
|
|
914
|
+
#coordinatingIncarnation: number | undefined;
|
|
915
|
+
/**
|
|
916
|
+
* Set when the caller supplied an explicit horizon. It waives both halves of the quarantine only
|
|
917
|
+
* for this coordinator's FIRST ownership interval, and `#ownershipHorizon` clears it at the first
|
|
918
|
+
* observed gap: the attestation behind it is "no previous incarnation of this process issued
|
|
919
|
+
* anything", which is a claim about process start and not about a sibling thread that coordinated
|
|
920
|
+
* while this one did not.
|
|
921
|
+
*/
|
|
922
|
+
#quarantineWaived: boolean;
|
|
923
|
+
/** Keys this node holds a delegation for. */
|
|
924
|
+
#delegations = new Map<unknown, Delegation>();
|
|
925
|
+
/** Grants received but not yet installed because their freshness barrier is still running. */
|
|
926
|
+
#pendingDelegations = new Map<unknown, PendingDelegation>();
|
|
927
|
+
/** Outbound requests whose grant token is not known yet, so an early recall cannot be lost. */
|
|
928
|
+
#pendingRequests = new Map<unknown, PendingRequest>();
|
|
929
|
+
/**
|
|
930
|
+
* Every live admission, by id, and the delegation answerable for it. Coordinator-level rather than
|
|
931
|
+
* per-delegation so a release can find its admission after the delegation was renewed or replaced.
|
|
932
|
+
*/
|
|
933
|
+
#admissions = new Map<number, Delegation>();
|
|
934
|
+
#nextAdmissionId = 1;
|
|
935
|
+
/**
|
|
936
|
+
* Admissions this thread obtained from the OWNER thread (harper-pro#852), keyed by a LOCAL id drawn
|
|
937
|
+
* from `#nextAdmissionId` — never the owner's id, so it cannot collide with a live local admission.
|
|
938
|
+
* This thread holds the handle (native key, staged writes, lease timer) while the delegation that
|
|
939
|
+
* authorizes it lives on the owner. Allocated lazily so a coordinator that never serves an off-owner
|
|
940
|
+
* lock pays nothing. An entry is kept after the caller unlocks, revoker included, until the handle's
|
|
941
|
+
* own lease runs out (owner-side §6 retention: an unlocked-but-staged write is still fenceable).
|
|
942
|
+
*/
|
|
943
|
+
#remoteAdmissions: Map<number, RemoteAdmission> | undefined;
|
|
944
|
+
/** Owner admission id → this thread's local id, so an inbound revoke (which names the owner id)
|
|
945
|
+
* reaches the right entry. */
|
|
946
|
+
#remoteByOwnerId: Map<number, number> | undefined;
|
|
947
|
+
/** Owner admission ids whose revoke arrived before `#acquireFromOwner` installed the entry, with the
|
|
948
|
+
* ack resolvers waiting on the eventual fence and the monotonic time they arrived. Drained when the
|
|
949
|
+
* entry installs (`#acquireFromOwner`); an entry whose acquire never lands (a revoke for an admission
|
|
950
|
+
* this thread already dropped) is resolved and swept once its wait exceeds a lease (`tick`). */
|
|
951
|
+
#pendingRemoteRevokes: Map<number, { resolvers: (() => void)[]; at: number }> | undefined;
|
|
952
|
+
/**
|
|
953
|
+
* Bumped every time every relayed admission is fenced wholesale (`fenceAllRemoteAdmissions`, i.e. the
|
|
954
|
+
* owner worker is gone). `#acquireFromOwner` samples it before it asks the owner and re-checks after
|
|
955
|
+
* the reply: a grant minted by an owner that has since been declared gone must NOT be installed, or
|
|
956
|
+
* this thread would start writing under an admission the replacement owner knows nothing about while
|
|
957
|
+
* that owner, starting empty, grants the same key to somebody else. Carried forward on handoff so a
|
|
958
|
+
* transport swap mid-acquire cannot reset it and re-open the window.
|
|
959
|
+
*/
|
|
960
|
+
#relayGeneration = 0;
|
|
961
|
+
#relayedAdmissions = 0;
|
|
962
|
+
/** Keys this node homes, and who holds each one. */
|
|
963
|
+
#grants = new Map<unknown, HomeGrant>();
|
|
964
|
+
/** Per-requester counts, so one peer cannot fill the home's table on its own. */
|
|
965
|
+
#grantsByRequester = new Map<string, number>();
|
|
966
|
+
/** Clean-handoff lineage outlives grants and is retained independently under its own cap. */
|
|
967
|
+
#dependencySets = new Map<unknown, LockDependencySet>();
|
|
968
|
+
#everDelegated = new DelegatedKeyFilter();
|
|
969
|
+
#freshnessGeneration: number | undefined;
|
|
970
|
+
/** False after any interval whose delegation history this coordinator could not have observed. */
|
|
971
|
+
#trustVirginKeys: boolean;
|
|
972
|
+
#counter = 0;
|
|
973
|
+
/**
|
|
974
|
+
* Closed coordinators must not keep admitting. `close()` sets this AND expires every delegation,
|
|
975
|
+
* because a handle already handed out checks only its own lease — clearing the table alone would
|
|
976
|
+
* let a successor coordinator (a re-registered transport) grant the same key with no lease time
|
|
977
|
+
* elapsed.
|
|
978
|
+
*/
|
|
979
|
+
#closed = false;
|
|
980
|
+
// A latched warning would make a permanently misrouted deployment look like a quiet cluster.
|
|
981
|
+
#droppedOffOwner = 0;
|
|
982
|
+
#lastOffOwnerWarn = 0;
|
|
983
|
+
|
|
984
|
+
constructor(options: LockCoordinatorOptions) {
|
|
985
|
+
if (!isNodeName(options.nodeId) || NON_DISTINCTIVE_NODE_NAMES.has(options.nodeId))
|
|
986
|
+
throw new LockUnavailableError(
|
|
987
|
+
`Cluster record locks need a distinctive node name to derive a key's home from, but this node identifies as "${options.nodeId}". Set node.hostname to this node's name in system.hdb_nodes.`
|
|
988
|
+
);
|
|
989
|
+
this.database = options.database;
|
|
990
|
+
this.table = options.table;
|
|
991
|
+
liveCoordinators.add(this);
|
|
992
|
+
this.nodeId = options.nodeId;
|
|
993
|
+
this.transport = options.transport;
|
|
994
|
+
this.#writeControl = options.writeControl;
|
|
995
|
+
this.#keyIdOf = options.keyIdOf;
|
|
996
|
+
this.#nextTimestamp = options.nextTimestamp;
|
|
997
|
+
this.#monotonic = options.monotonic ?? (() => performance.now());
|
|
998
|
+
this.#skewMs = options.skewMs ?? LOCK_LEASE_SKEW_MS;
|
|
999
|
+
this.#autoTick = options.autoTick !== false;
|
|
1000
|
+
// Anchored at construction, which is the only instant core can prove nothing else was granting
|
|
1001
|
+
// under: process start is wrong (`performance.now()` is process-wide inside a worker too, so a
|
|
1002
|
+
// replacement coordinating thread would read an uptime far past it) and so is thread start (a
|
|
1003
|
+
// thread can take coordination ownership long after it booted). `adopt` and the retirement
|
|
1004
|
+
// record below waive or carry the horizon wherever a predecessor's authority is actually known.
|
|
1005
|
+
this.#quarantineWaived = options.grantableAfterMono !== undefined;
|
|
1006
|
+
this.#trustVirginKeys = this.#quarantineWaived;
|
|
1007
|
+
// Ownership observed here, not lazily on the first grant: a coordinator built while this thread
|
|
1008
|
+
// already coordinates has owned it since construction, and the construction horizon covers that.
|
|
1009
|
+
// Leaving it unset until a grant would date ownership from the grant and re-quarantine a reload.
|
|
1010
|
+
try {
|
|
1011
|
+
if (options.transport.ownsCoordination()) {
|
|
1012
|
+
this.#ownedSinceMono = this.#monotonic();
|
|
1013
|
+
// The incarnation it has been coordinating under, so the first grant does not read its own
|
|
1014
|
+
// construction as a takeover. Undefined when there is no map yet, which `#ownershipHorizon`
|
|
1015
|
+
// treats as a takeover — the conservative direction.
|
|
1016
|
+
this.#coordinatingIncarnation = options.transport.homeMap(options.database)?.homeIncarnation;
|
|
1017
|
+
}
|
|
1018
|
+
} catch {
|
|
1019
|
+
// A transport that cannot answer yet is not owning yet; `#ownershipHorizon` will observe it.
|
|
1020
|
+
}
|
|
1021
|
+
this.#grantableAfterMono = options.grantableAfterMono ?? this.#monotonic() + DELEGATION_LEASE_MS + this.#skewMs;
|
|
1022
|
+
options.adopt?.handOffTo(this);
|
|
1023
|
+
const retired = retiredCoordinators.get(this.#retirementKey());
|
|
1024
|
+
if (retired) {
|
|
1025
|
+
this.#counter = Math.max(this.#counter, retired.counter);
|
|
1026
|
+
this.#grantableAfterMono = Math.max(this.#grantableAfterMono, retired.grantableAfterMono);
|
|
1027
|
+
// A close without a successor discarded the key filter and retained dependency sets. An
|
|
1028
|
+
// explicit cold-start waiver on the replacement says nothing about that discarded history.
|
|
1029
|
+
this.#trustVirginKeys = false;
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/**
|
|
1034
|
+
* Move this coordinator's live authority to the coordinator replacing it. Called from the
|
|
1035
|
+
* successor's constructor, before `close()`, so nothing is dropped in between. Both coordinators
|
|
1036
|
+
* run in the same thread for the same node, so the delegations and grants are still this node's —
|
|
1037
|
+
* only the transport object underneath them changed.
|
|
1038
|
+
*/
|
|
1039
|
+
handOffTo(successor: LockCoordinator): void {
|
|
1040
|
+
if (this.#handedOff) return;
|
|
1041
|
+
this.#handedOff = true;
|
|
1042
|
+
// Kept so an acquisition still awaiting a home's reply can install it on whoever holds this
|
|
1043
|
+
// node's authority when the reply lands, rather than on a coordinator nothing consults.
|
|
1044
|
+
this.#successor = successor;
|
|
1045
|
+
for (const [keyId, delegation] of this.#delegations) successor.#delegations.set(keyId, delegation);
|
|
1046
|
+
for (const [keyId, pending] of this.#pendingDelegations) successor.#pendingDelegations.set(keyId, pending);
|
|
1047
|
+
for (const [keyId, request] of this.#pendingRequests) successor.#pendingRequests.set(keyId, request);
|
|
1048
|
+
for (const [keyId, grant] of this.#grants) successor.#grants.set(keyId, grant);
|
|
1049
|
+
for (const [requester, count] of this.#grantsByRequester) successor.#grantsByRequester.set(requester, count);
|
|
1050
|
+
// The admission index moves with the delegations it points into. Without it a handle admitted
|
|
1051
|
+
// on the predecessor could never be released through the successor — `release` addresses the
|
|
1052
|
+
// admission, so the entry would sit on the delegation forever and no recall could drain it.
|
|
1053
|
+
for (const [admissionId, delegation] of this.#admissions) successor.#admissions.set(admissionId, delegation);
|
|
1054
|
+
// Remote admissions (harper-pro#852) move with their handles: this thread still holds the native
|
|
1055
|
+
// key and the staged write for each, so the successor must be the one a later `release` or a
|
|
1056
|
+
// relayed `revoke` reaches. Their local ids stay valid (the successor's own `#nextAdmissionId` is
|
|
1057
|
+
// carried forward below), and the owner-id index and any not-yet-installed revokes move with them.
|
|
1058
|
+
if (this.#remoteAdmissions) {
|
|
1059
|
+
const into = (successor.#remoteAdmissions ??= new Map());
|
|
1060
|
+
for (const [localId, remote] of this.#remoteAdmissions) into.set(localId, remote);
|
|
1061
|
+
const ownerIndex = (successor.#remoteByOwnerId ??= new Map());
|
|
1062
|
+
for (const [ownerId, localId] of this.#remoteByOwnerId ?? []) ownerIndex.set(ownerId, localId);
|
|
1063
|
+
this.#remoteAdmissions = undefined;
|
|
1064
|
+
this.#remoteByOwnerId = undefined;
|
|
1065
|
+
}
|
|
1066
|
+
if (this.#pendingRemoteRevokes) {
|
|
1067
|
+
const into = (successor.#pendingRemoteRevokes ??= new Map());
|
|
1068
|
+
for (const [ownerId, entry] of this.#pendingRemoteRevokes) {
|
|
1069
|
+
const existing = into.get(ownerId);
|
|
1070
|
+
if (existing) existing.resolvers.push(...entry.resolvers);
|
|
1071
|
+
else into.set(ownerId, entry);
|
|
1072
|
+
}
|
|
1073
|
+
this.#pendingRemoteRevokes = undefined;
|
|
1074
|
+
}
|
|
1075
|
+
successor.#relayedAdmissions += this.#relayedAdmissions;
|
|
1076
|
+
// Never backwards: an acquire that sampled the predecessor must still see a bump the successor
|
|
1077
|
+
// (or the predecessor) already recorded, so a swap mid-acquire cannot re-open the orphan window.
|
|
1078
|
+
successor.#relayGeneration = Math.max(successor.#relayGeneration, this.#relayGeneration);
|
|
1079
|
+
// Neither counter may restart. A repeated token would compare equal to one the predecessor
|
|
1080
|
+
// already issued for a different delegation; a repeated admission id would address the wrong
|
|
1081
|
+
// admission in the map just carried over.
|
|
1082
|
+
successor.#counter = Math.max(successor.#counter, this.#counter);
|
|
1083
|
+
successor.#nextAdmissionId = Math.max(successor.#nextAdmissionId, this.#nextAdmissionId);
|
|
1084
|
+
// The predecessor's horizon EXACTLY, not the successor's freshly computed one. Adoption means the
|
|
1085
|
+
// successor now knows everything the predecessor knew, so it faces the same cold-start hazard and
|
|
1086
|
+
// no more: recomputing from its own construction would quarantine a node for a full delegation
|
|
1087
|
+
// lease on every transport reload, and clearing it would let a swap inside the window grant over
|
|
1088
|
+
// an unseen predecessor incarnation.
|
|
1089
|
+
successor.#grantableAfterMono = this.#grantableAfterMono;
|
|
1090
|
+
// And the ownership clock: a transport reload does not change which thread coordinates, so the
|
|
1091
|
+
// successor inherits how long this one has owned it rather than starting a fresh interval. The
|
|
1092
|
+
// waiver rides along with it and in the same direction: the successor reads `grantableAfterMono`
|
|
1093
|
+
// off the new transport and would otherwise re-waive a horizon this coordinator had already lost
|
|
1094
|
+
// to a takeover.
|
|
1095
|
+
//
|
|
1096
|
+
// Only what this coordinator actually OBSERVED may override that, though. `undefined` on either
|
|
1097
|
+
// of these means "never saw it" — a coordinator built on a non-owning thread, or before a map
|
|
1098
|
+
// was available — and the successor has just read the new transport, which is not less current.
|
|
1099
|
+
// Carrying the blank over it re-armed a quarantine on a node that had never stopped coordinating.
|
|
1100
|
+
if (this.#ownedSinceMono !== undefined) {
|
|
1101
|
+
successor.#ownedSinceMono = this.#ownedSinceMono;
|
|
1102
|
+
successor.#quarantineWaived = this.#quarantineWaived;
|
|
1103
|
+
}
|
|
1104
|
+
if (this.#coordinatingIncarnation !== undefined) successor.#coordinatingIncarnation = this.#coordinatingIncarnation;
|
|
1105
|
+
successor.#dependencySets = this.#dependencySets;
|
|
1106
|
+
successor.#everDelegated.copyFrom(this.#everDelegated);
|
|
1107
|
+
successor.#freshnessGeneration = this.#freshnessGeneration;
|
|
1108
|
+
successor.#trustVirginKeys = this.#trustVirginKeys;
|
|
1109
|
+
this.#dependencySets = new Map();
|
|
1110
|
+
this.#everDelegated.clear();
|
|
1111
|
+
this.#delegations.clear();
|
|
1112
|
+
this.#pendingDelegations.clear();
|
|
1113
|
+
this.#pendingRequests.clear();
|
|
1114
|
+
this.#grants.clear();
|
|
1115
|
+
this.#grantsByRequester.clear();
|
|
1116
|
+
this.#admissions.clear();
|
|
1117
|
+
if (
|
|
1118
|
+
successor.#delegations.size > 0 ||
|
|
1119
|
+
successor.#pendingDelegations.size > 0 ||
|
|
1120
|
+
successor.#grants.size > 0 ||
|
|
1121
|
+
(successor.#remoteAdmissions?.size ?? 0) > 0 ||
|
|
1122
|
+
(successor.#pendingRemoteRevokes?.size ?? 0) > 0
|
|
1123
|
+
)
|
|
1124
|
+
successor.#startTicking();
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
/**
|
|
1128
|
+
* For `cluster_status`: delegations held, delegations issued, live admissions, admissions still
|
|
1129
|
+
* revocable, misrouted calls. `revocable` counts the ones an unlock left fenceable as well, so the
|
|
1130
|
+
* retention `#pruneAdmissions` bounds is visible rather than inferred from `admitted`.
|
|
1131
|
+
*/
|
|
1132
|
+
get stats(): {
|
|
1133
|
+
delegations: number;
|
|
1134
|
+
granted: number;
|
|
1135
|
+
admitted: number;
|
|
1136
|
+
revocable: number;
|
|
1137
|
+
droppedOffOwner: number;
|
|
1138
|
+
relayedAdmissions: number;
|
|
1139
|
+
} {
|
|
1140
|
+
let admitted = 0;
|
|
1141
|
+
let revocable = 0;
|
|
1142
|
+
for (const delegation of this.#delegations.values()) {
|
|
1143
|
+
admitted += delegation.holding;
|
|
1144
|
+
revocable += delegation.admissions.size;
|
|
1145
|
+
}
|
|
1146
|
+
return {
|
|
1147
|
+
delegations: this.#delegations.size,
|
|
1148
|
+
granted: this.#grants.size,
|
|
1149
|
+
admitted,
|
|
1150
|
+
revocable,
|
|
1151
|
+
droppedOffOwner: this.#droppedOffOwner,
|
|
1152
|
+
// Admissions this thread has obtained from the owner worker (harper-pro#852), cumulative
|
|
1153
|
+
// rather than current: what it makes visible in `cluster_status` is that off-owner `lock()`s
|
|
1154
|
+
// are being served here at all.
|
|
1155
|
+
relayedAdmissions: this.#relayedAdmissions,
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
/**
|
|
1160
|
+
* Admit a critical section for a key whose native lock this thread already holds. Resolves with
|
|
1161
|
+
* the stamp and the monotonic reading the lease runs from; rejects 423 when no delegation could be
|
|
1162
|
+
* obtained in time, or 503 when the guarantee cannot be established at all.
|
|
1163
|
+
*
|
|
1164
|
+
* The amortization is the first branch: a live, un-recalled delegation with enough time left costs
|
|
1165
|
+
* zero cluster messages.
|
|
1166
|
+
*/
|
|
1167
|
+
acquire(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
|
|
1168
|
+
return this.#acquire(key, leaseMs, waitMs);
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
async #acquire(key: any, leaseMs: number, waitMs: number, observed?: LastCompletedReply): Promise<LockRound> {
|
|
1172
|
+
// `Table.lock()` captures a coordinator and only reaches here after the native key lock, which
|
|
1173
|
+
// can wait the caller's whole timeout — long enough for a transport swap to close what it
|
|
1174
|
+
// captured. Authority moved to the successor rather than away, so run there instead of
|
|
1175
|
+
// rejecting a caller that is already holding the key.
|
|
1176
|
+
const authority = this.#authority();
|
|
1177
|
+
if (authority !== this) return authority.#acquire(key, leaseMs, waitMs, observed);
|
|
1178
|
+
if (this.#closed) throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1179
|
+
if (!this.transport.ownsCoordination()) {
|
|
1180
|
+
// Off the coordinating thread. A transport that can relay obtains the admission from the
|
|
1181
|
+
// owner and installs the handle's revoker here (harper-pro#852); one that cannot fails
|
|
1182
|
+
// closed, the historical contract.
|
|
1183
|
+
// BOTH halves or neither. A transport that could acquire but not release would route locks and
|
|
1184
|
+
// then never forward an unlock, leaving every admission `holding` on the owner for its full
|
|
1185
|
+
// lease and stalling peer recalls — worse than the honest 503 below.
|
|
1186
|
+
if (this.transport.acquireOnOwner && this.transport.releaseOnOwner)
|
|
1187
|
+
return this.#acquireFromOwner(key, leaseMs, waitMs);
|
|
1188
|
+
throw new LockUnavailableError(
|
|
1189
|
+
'Cluster record lock coordination is not owned by this worker thread; retry so the request reaches the coordinating thread'
|
|
1190
|
+
);
|
|
1191
|
+
}
|
|
1192
|
+
const keyId = this.#keyIdOf(key);
|
|
1193
|
+
const deadlineMono = this.#monotonic() + waitMs;
|
|
1194
|
+
let lastCompleted = observed;
|
|
1195
|
+
|
|
1196
|
+
acquisition: for (;;) {
|
|
1197
|
+
const homeMap = this.transport.homeMap(this.database);
|
|
1198
|
+
// No agreed map means no agreed ring, and a ring guessed from whoever looks reachable is
|
|
1199
|
+
// exactly the asymmetric-partition failure a single arbiter exists to remove.
|
|
1200
|
+
if (!homeMap || !this.#generationIsCurrent(homeMap.generation))
|
|
1201
|
+
throw new LockUnavailableError(
|
|
1202
|
+
`No agreed record lock home map for ${this.database}; cluster record locks are unavailable until one is established`
|
|
1203
|
+
);
|
|
1204
|
+
const delegation = this.#liveDelegation(keyId, leaseMs, homeMap.generation);
|
|
1205
|
+
if (delegation) return this.#admit(delegation, leaseMs);
|
|
1206
|
+
|
|
1207
|
+
const home = homeFor(this.#ringKey(keyId), homeMap.homes);
|
|
1208
|
+
if (!home)
|
|
1209
|
+
throw new LockUnavailableError(`The record lock home map for ${this.database} names no nodes to home a key on`);
|
|
1210
|
+
|
|
1211
|
+
// Anchored before the send: the home starts its own clock when it grants, so measuring the
|
|
1212
|
+
// delegation from the reply's arrival would hand a delayed reply more time than the home is
|
|
1213
|
+
// holding the key for.
|
|
1214
|
+
const requestedAtMono = this.#monotonic();
|
|
1215
|
+
const pendingRequest: PendingRequest = {};
|
|
1216
|
+
this.#pendingRequests.set(keyId, pendingRequest);
|
|
1217
|
+
let reply: DelegationReply;
|
|
1218
|
+
try {
|
|
1219
|
+
reply =
|
|
1220
|
+
home === this.nodeId
|
|
1221
|
+
? this.#grantLocally(keyId, key, homeMap, leaseMs)
|
|
1222
|
+
: await this.#requestRemotely(home, keyId, key, homeMap, leaseMs, deadlineMono);
|
|
1223
|
+
} catch (error) {
|
|
1224
|
+
const requestAuthority = this.#authority();
|
|
1225
|
+
if (requestAuthority.#pendingRequests.get(keyId) === pendingRequest)
|
|
1226
|
+
requestAuthority.#pendingRequests.delete(keyId);
|
|
1227
|
+
throw error;
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
// A transport swap can land while a request is in flight. The grant is authority for this
|
|
1231
|
+
// NODE, and the successor is this node now — installing it here would leave a delegation
|
|
1232
|
+
// nothing consults, admitting a caller that no recall can reach.
|
|
1233
|
+
let authority = this.#authority();
|
|
1234
|
+
if (authority.#pendingRequests.get(keyId) === pendingRequest) authority.#pendingRequests.delete(keyId);
|
|
1235
|
+
if (authority.#closed)
|
|
1236
|
+
throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1237
|
+
|
|
1238
|
+
const replyDependencies =
|
|
1239
|
+
reply.dependencies === null ? null : normalizeDependencies(reply.dependencies, homeMap.homes);
|
|
1240
|
+
if (
|
|
1241
|
+
reply.granted &&
|
|
1242
|
+
isFencingToken(reply.token) &&
|
|
1243
|
+
isDuration(reply.leaseMs, MIN_LOCK_LEASE_MS, DELEGATION_LEASE_MS) &&
|
|
1244
|
+
replyDependencies !== undefined
|
|
1245
|
+
) {
|
|
1246
|
+
const recalledBeforeReply =
|
|
1247
|
+
pendingRequest.recalledToken !== undefined && compareTokens(pendingRequest.recalledToken, reply.token) === 0;
|
|
1248
|
+
const existing = authority.#delegations.get(keyId);
|
|
1249
|
+
if (
|
|
1250
|
+
existing &&
|
|
1251
|
+
!existing.recalled &&
|
|
1252
|
+
existing.token[0] === reply.token[0] &&
|
|
1253
|
+
compareTokens(existing.token, reply.token) < 0
|
|
1254
|
+
) {
|
|
1255
|
+
// This is a continuous renewal of authority this node already made fresh. No other
|
|
1256
|
+
// delegate could have held the key, so repeating the barrier — especially recovery —
|
|
1257
|
+
// buys no freshness. The home still returns its requirement for the case where the
|
|
1258
|
+
// requester lost local state and has no existing delegation to prove continuity.
|
|
1259
|
+
const renewed = authority.#installDelegation(
|
|
1260
|
+
keyId,
|
|
1261
|
+
key,
|
|
1262
|
+
reply.token,
|
|
1263
|
+
reply.leaseMs,
|
|
1264
|
+
requestedAtMono,
|
|
1265
|
+
existing.dependencies
|
|
1266
|
+
);
|
|
1267
|
+
if (renewed && recalledBeforeReply) {
|
|
1268
|
+
// The renewed delegation inherits every admission from the old token. Drain those
|
|
1269
|
+
// admissions through the ordinary recall path; surrendering directly would revoke
|
|
1270
|
+
// their handles and clear the home's grant while their critical sections still run.
|
|
1271
|
+
await authority.onDelegationRecall({ key, token: reply.token });
|
|
1272
|
+
if (authority.#closed)
|
|
1273
|
+
throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1274
|
+
continue acquisition;
|
|
1275
|
+
}
|
|
1276
|
+
if (renewed && renewed.expiresMono - authority.#monotonic() >= leaseMs)
|
|
1277
|
+
return authority.#admit(renewed, leaseMs);
|
|
1278
|
+
continue acquisition;
|
|
1279
|
+
}
|
|
1280
|
+
const pending = createPendingDelegation(key, reply.token, recalledBeforeReply);
|
|
1281
|
+
authority.#pendingDelegations.set(keyId, pending);
|
|
1282
|
+
let dependencies: LockDependencySet;
|
|
1283
|
+
try {
|
|
1284
|
+
let requirement = replyDependencies;
|
|
1285
|
+
for (;;) {
|
|
1286
|
+
const barrierAuthority = authority;
|
|
1287
|
+
dependencies = await barrierAuthority.#establishFreshness(
|
|
1288
|
+
key,
|
|
1289
|
+
requirement,
|
|
1290
|
+
homeMap.homes,
|
|
1291
|
+
deadlineMono,
|
|
1292
|
+
pending
|
|
1293
|
+
);
|
|
1294
|
+
authority = this.#authority();
|
|
1295
|
+
if (authority === barrierAuthority) break;
|
|
1296
|
+
requirement = dependencies;
|
|
1297
|
+
}
|
|
1298
|
+
} catch (error) {
|
|
1299
|
+
authority = this.#authority();
|
|
1300
|
+
if (authority.#pendingDelegations.get(keyId) === pending) authority.#pendingDelegations.delete(keyId);
|
|
1301
|
+
await authority.#releaseUnclaimedGrant(key, reply.token);
|
|
1302
|
+
if (error === PENDING_DELEGATION_RECALLED) {
|
|
1303
|
+
if (authority.#closed)
|
|
1304
|
+
throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1305
|
+
continue acquisition;
|
|
1306
|
+
}
|
|
1307
|
+
throw new LockUnavailableError(
|
|
1308
|
+
`Could not establish successor freshness for ${this.database}.${this.table}: ${(error as Error)?.message ?? error}`
|
|
1309
|
+
);
|
|
1310
|
+
}
|
|
1311
|
+
const currentPending = authority.#pendingDelegations.get(keyId);
|
|
1312
|
+
if (currentPending === pending) authority.#pendingDelegations.delete(keyId);
|
|
1313
|
+
if (currentPending !== pending || pending.recalled) {
|
|
1314
|
+
await authority.#releaseUnclaimedGrant(key, reply.token);
|
|
1315
|
+
if (authority.#closed)
|
|
1316
|
+
throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1317
|
+
continue;
|
|
1318
|
+
}
|
|
1319
|
+
// The generation can advance across the await. A grant minted under a superseded one is
|
|
1320
|
+
// authority for a ring that no longer exists: the key may be homed elsewhere now, and that
|
|
1321
|
+
// home can already have granted it to another node. `#liveDelegation` rejects a stale
|
|
1322
|
+
// token on the NEXT pass, which is too late — this pass would have admitted on it first.
|
|
1323
|
+
if (reply.token[0] !== authority.transport.homeMap(this.database)?.generation) {
|
|
1324
|
+
// Hand it back rather than let the old home hold a key nobody is using for a full lease.
|
|
1325
|
+
authority.#releaseUnclaimedGrant(key, reply.token);
|
|
1326
|
+
} else {
|
|
1327
|
+
const installed = authority.#installDelegation(
|
|
1328
|
+
keyId,
|
|
1329
|
+
key,
|
|
1330
|
+
reply.token,
|
|
1331
|
+
reply.leaseMs,
|
|
1332
|
+
requestedAtMono,
|
|
1333
|
+
dependencies
|
|
1334
|
+
);
|
|
1335
|
+
// A reply that outlived its own delegation grants nothing; fall through and ask again
|
|
1336
|
+
// rather than admitting on authority the home has already expired.
|
|
1337
|
+
if (installed && !installed.recalled && installed.expiresMono - authority.#monotonic() >= leaseMs)
|
|
1338
|
+
return authority.#admit(installed, leaseMs);
|
|
1339
|
+
// A long barrier can consume enough of the delegation that the requested lease no longer
|
|
1340
|
+
// fits. Surrender the unused installed delegation before retrying; otherwise the home renews
|
|
1341
|
+
// it in place on every pass and contenders wait for its full deadline.
|
|
1342
|
+
if (installed && compareTokens(installed.token, reply.token) === 0 && installed.holding === 0)
|
|
1343
|
+
await authority.#surrender(keyId, installed);
|
|
1344
|
+
else if (!installed) await authority.#releaseUnclaimedGrant(key, reply.token);
|
|
1345
|
+
}
|
|
1346
|
+
} else if (reply.granted) {
|
|
1347
|
+
if (isFencingToken(reply.token)) authority.#releaseUnclaimedGrant(key, reply.token);
|
|
1348
|
+
throw new LockUnavailableError(
|
|
1349
|
+
`The home node for this key on ${this.database}.${this.table} returned a delegation with no usable token or freshness requirement`
|
|
1350
|
+
);
|
|
1351
|
+
}
|
|
1352
|
+
if (reply.reason === 'capacity')
|
|
1353
|
+
throw new LockUnavailableError(`Too many record lock delegations in flight on ${this.database}`);
|
|
1354
|
+
if (reply.reason === 'quarantine')
|
|
1355
|
+
throw new LockUnavailableError(
|
|
1356
|
+
`The home node for this key on ${this.database}.${this.table} restarted and cannot grant until the delegations its previous incarnation issued have expired`
|
|
1357
|
+
);
|
|
1358
|
+
// Neither of these can be waited out inside a `lock()` timeout, and retrying them spends the
|
|
1359
|
+
// caller's whole budget holding the native key only to answer 423 for a key nobody holds.
|
|
1360
|
+
if (reply.reason === 'generation')
|
|
1361
|
+
throw new LockUnavailableError(
|
|
1362
|
+
`The home node for this key on ${this.database}.${this.table} holds record lock home map generation ${reply.generation ?? 'unknown'} and this node holds ${homeMap.generation}; cluster record locks are unavailable until they agree`
|
|
1363
|
+
);
|
|
1364
|
+
if (reply.reason === 'unknown-node')
|
|
1365
|
+
throw new LockUnavailableError(
|
|
1366
|
+
`This node is not named in the record lock home map for ${this.database}, so it can neither home a key nor take a cluster lock on one`
|
|
1367
|
+
);
|
|
1368
|
+
if (reply.reason === 'not-home')
|
|
1369
|
+
// The home disagrees about the ring. Re-reading the map on the next pass is the fix, and it
|
|
1370
|
+
// converges if our copy is the stale one — so unlike the denials above this one is worth
|
|
1371
|
+
// retrying. If it is NOT stale (two maps under one generation number) it never converges,
|
|
1372
|
+
// which the terminal answer below is what handles.
|
|
1373
|
+
warnOnce('record lock home disagreed about the ring', { database: this.database, table: this.table });
|
|
1374
|
+
|
|
1375
|
+
if (reply.reason !== 'timeout') lastCompleted = { reply, home, generation: homeMap.generation };
|
|
1376
|
+
const remaining = deadlineMono - this.#monotonic();
|
|
1377
|
+
const retryAfterMs = reply.retryAfterMs ?? 25;
|
|
1378
|
+
// A home this node IS costs nothing to ask again — `#grantLocally` is synchronous, so a release
|
|
1379
|
+
// landing in the backoff is still grantable at the deadline. A remote home is not: its request
|
|
1380
|
+
// would go out with the leftover budget and could only return this node's own `timeout`.
|
|
1381
|
+
const exhausted = home === this.nodeId ? remaining <= 0 : remaining <= retryAfterMs;
|
|
1382
|
+
if (!exhausted) await delay(Math.min(retryAfterMs, remaining)).promise;
|
|
1383
|
+
if (this.#closed) {
|
|
1384
|
+
// Same swap, landing in the backoff instead. The successor inherits both the remaining wait
|
|
1385
|
+
// and what this one saw, since a successor exhausted on arrival has nothing of its own.
|
|
1386
|
+
const successor = this.#authority();
|
|
1387
|
+
if (successor === this)
|
|
1388
|
+
throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1389
|
+
return successor.#acquire(key, leaseMs, Math.max(0, deadlineMono - this.#monotonic()), lastCompleted);
|
|
1390
|
+
}
|
|
1391
|
+
if (!exhausted) continue acquisition;
|
|
1392
|
+
// Only an observation made under the generation that is current NOW still describes the key —
|
|
1393
|
+
// this pass's reply included, since a generation can be activated while the probe that ended
|
|
1394
|
+
// the wait is still in flight. Hence a fresh read rather than the copy this pass started from.
|
|
1395
|
+
const currentGeneration = this.transport.homeMap(this.database)?.generation;
|
|
1396
|
+
const carried =
|
|
1397
|
+
lastCompleted?.home === home && lastCompleted.generation === currentGeneration
|
|
1398
|
+
? lastCompleted.reply
|
|
1399
|
+
: undefined;
|
|
1400
|
+
const terminal = carried ?? (homeMap.generation === currentGeneration ? reply : undefined);
|
|
1401
|
+
// Only `contended` may end as 423: it is the one answer that says another node holds the key,
|
|
1402
|
+
// and reporting contention for anything else sends the caller to retry a condition no wait
|
|
1403
|
+
// outlasts. A wait can see `contended` and still end 503 — the rule is what the home said LAST.
|
|
1404
|
+
if (terminal?.reason === 'contended') throw new ClientError('Record is locked and was not released in time', 423);
|
|
1405
|
+
throw new LockUnavailableError(
|
|
1406
|
+
`Could not establish a cluster record lock on ${this.database}.${this.table} within the wait: ${describeExhaustedWait(terminal, currentGeneration !== undefined)}`
|
|
1407
|
+
);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
/**
|
|
1412
|
+
* Obtain an admission from the owner worker for a `lock()` served on a non-owner thread
|
|
1413
|
+
* (harper-pro#852), recorded under a LOCAL id so `registerAdmission`/`release` can never collide it
|
|
1414
|
+
* with a live local admission. `acquireOnOwner` is bounded by the transport, inside the caller's
|
|
1415
|
+
* `waitMs`; the race here is core's own backstop against a transport that never answers, so `lock()`
|
|
1416
|
+
* cannot hang far past the wait it was given.
|
|
1417
|
+
*/
|
|
1418
|
+
async #acquireFromOwner(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
|
|
1419
|
+
// The transport bounds this wait (harper-pro's acquire has its own `waitMs`-scaled timeout); the
|
|
1420
|
+
// core-side race is a last-resort backstop set strictly beyond that bound — the transport is asked
|
|
1421
|
+
// for `waitMs - hop` and core's deadline is `waitMs + REMOTE_ACQUIRE_BACKSTOP_MS` — so it fires only
|
|
1422
|
+
// if the transport never returns, never ahead of the transport's own timeout. Without it a wedged
|
|
1423
|
+
// owner would hang `lock()` past its `waitMs`.
|
|
1424
|
+
// Sampled BEFORE the request goes out, on the authority that will install the result.
|
|
1425
|
+
const startGeneration = this.#authority().#relayGeneration;
|
|
1426
|
+
// The hop allowance comes OUT of the caller's budget, never on top of it: `lock()` holds the native
|
|
1427
|
+
// key for this whole wait, so overshooting `waitMs` blocks every other worker on the key for the
|
|
1428
|
+
// overshoot. The transport is asked for the reduced wait and answers within the caller's budget;
|
|
1429
|
+
// core's deadline is the caller's `waitMs` plus a small net that fires only if the transport never
|
|
1430
|
+
// answers at all.
|
|
1431
|
+
const hop = Math.min(REMOTE_ACQUIRE_HOP_MS, Math.floor(waitMs / 2));
|
|
1432
|
+
// A transport that throws SYNCHRONOUSLY (a sibling port already gone) must still reach the
|
|
1433
|
+
// normalization below: escaping raw gives the caller a 500 where it needs the retryable 503.
|
|
1434
|
+
// `#beginRecall` wraps its transport call for the same reason.
|
|
1435
|
+
const acquire = Promise.resolve().then(() =>
|
|
1436
|
+
this.transport.acquireOnOwner!(this.database, this.table, key, leaseMs, waitMs - hop)
|
|
1437
|
+
);
|
|
1438
|
+
let backstopWon = false;
|
|
1439
|
+
// If the backstop wins the race, the owner may still grant afterward: release that grant back so it
|
|
1440
|
+
// does not sit `holding` on the owner for its whole lease. (The transport releases a late reply too;
|
|
1441
|
+
// this closes the case where core's backstop fired first.)
|
|
1442
|
+
acquire.then(
|
|
1443
|
+
(round) => {
|
|
1444
|
+
if (backstopWon) this.#authority().#releaseOnOwnerSafely(key, round.admissionId);
|
|
1445
|
+
},
|
|
1446
|
+
() => {}
|
|
1447
|
+
);
|
|
1448
|
+
const round = await withDeadline(
|
|
1449
|
+
acquire,
|
|
1450
|
+
waitMs + REMOTE_ACQUIRE_BACKSTOP_MS,
|
|
1451
|
+
'the coordinating worker did not answer'
|
|
1452
|
+
).catch((error) => {
|
|
1453
|
+
backstopWon = true;
|
|
1454
|
+
throw error instanceof LockUnavailableError
|
|
1455
|
+
? error
|
|
1456
|
+
: new LockUnavailableError(
|
|
1457
|
+
`Could not obtain a cluster record lock on ${this.database}.${this.table} from the coordinating worker: ${(error as Error)?.message ?? error}`
|
|
1458
|
+
);
|
|
1459
|
+
});
|
|
1460
|
+
const authority = this.#authority();
|
|
1461
|
+
if (authority.#closed) {
|
|
1462
|
+
// The coordinator closed while the owner was granting. The owner still holds this admission;
|
|
1463
|
+
// hand it straight back rather than leaving it outstanding for its whole lease.
|
|
1464
|
+
authority.#releaseOnOwnerSafely(key, round.admissionId);
|
|
1465
|
+
throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
|
|
1466
|
+
}
|
|
1467
|
+
if (authority.#relayGeneration !== startGeneration) {
|
|
1468
|
+
// Every relayed admission was fenced while this grant was in flight: the owner that minted it
|
|
1469
|
+
// has been declared gone, and the replacement starts with no record of it. Installing it now
|
|
1470
|
+
// would let this thread write under an admission the new owner can neither see nor recall,
|
|
1471
|
+
// while that owner grants the same key to another worker — two writers. Fail closed and hand
|
|
1472
|
+
// the grant back; the caller retries against the new owner.
|
|
1473
|
+
authority.#releaseOnOwnerSafely(key, round.admissionId);
|
|
1474
|
+
throw new LockUnavailableError(
|
|
1475
|
+
'The record lock coordinating worker changed while this lock was being granted; retry against the new owner'
|
|
1476
|
+
);
|
|
1477
|
+
}
|
|
1478
|
+
// A live entry already keyed to this owner id means the owner reused an id its counter had already
|
|
1479
|
+
// issued — only possible if a replacement owner restarted its sequence. The old entry is from that
|
|
1480
|
+
// prior owner incarnation and its delegation is gone: fence it fail-closed before the new one takes
|
|
1481
|
+
// the id, so a later revoke for this id can never address the stale handle.
|
|
1482
|
+
const collidingLocalId = authority.#remoteByOwnerId?.get(round.admissionId);
|
|
1483
|
+
if (collidingLocalId !== undefined) {
|
|
1484
|
+
const stale = authority.#remoteAdmissions?.get(collidingLocalId);
|
|
1485
|
+
if (stale) {
|
|
1486
|
+
fireRevokeAndForget(stale.revoke);
|
|
1487
|
+
authority.#dropRemoteAdmission(collidingLocalId, stale);
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
const localId = authority.#nextAdmissionId++;
|
|
1491
|
+
const entry: RemoteAdmission = {
|
|
1492
|
+
ownerAdmissionId: round.admissionId,
|
|
1493
|
+
key,
|
|
1494
|
+
revoke: noRevoke,
|
|
1495
|
+
expiresMono: round.mintedMono + leaseMs,
|
|
1496
|
+
revoked: false,
|
|
1497
|
+
fenceWaiters: [],
|
|
1498
|
+
};
|
|
1499
|
+
(authority.#remoteAdmissions ??= new Map()).set(localId, entry);
|
|
1500
|
+
(authority.#remoteByOwnerId ??= new Map()).set(round.admissionId, localId);
|
|
1501
|
+
// A revoke that raced ahead of this install (the owner registered its revoker before it posted
|
|
1502
|
+
// the grant reply, so a recall in that window fired first): apply it now so the ack the owner is
|
|
1503
|
+
// waiting on cannot resolve before the handle is fenced.
|
|
1504
|
+
const pending = authority.#pendingRemoteRevokes?.get(round.admissionId);
|
|
1505
|
+
if (pending) {
|
|
1506
|
+
authority.#pendingRemoteRevokes!.delete(round.admissionId);
|
|
1507
|
+
entry.revoked = true;
|
|
1508
|
+
entry.fenceWaiters.push(...pending.resolvers);
|
|
1509
|
+
}
|
|
1510
|
+
authority.#relayedAdmissions++;
|
|
1511
|
+
// Tick so an entry whose caller never releases (its lease elapses instead) is still pruned.
|
|
1512
|
+
authority.#startTicking();
|
|
1513
|
+
// The LOCAL id is what the handle registers and releases against; the round the owner minted
|
|
1514
|
+
// still carries its own id, which only the owner-facing release/revoke messages use.
|
|
1515
|
+
return { tsR: round.tsR, mintedMono: round.mintedMono, admissionId: localId };
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
/**
|
|
1519
|
+
* Fence a remote admission's handle on this thread, driven by a recall or surrender on the owner
|
|
1520
|
+
* (harper-pro#852), addressed by the OWNER's admission id. Resolves once the handle is provably
|
|
1521
|
+
* fenced — its `revokeLease` has run — so the owner's ack cannot claim a fence the handle has not
|
|
1522
|
+
* taken; the owner waits on this before writing the release. A revoke that arrives before the handle
|
|
1523
|
+
* is installed, or before `registerAdmission` supplies the real revoker, resolves only when the fence
|
|
1524
|
+
* finally lands. A revoker that throws, or whose promise rejects, fails this promise rather than
|
|
1525
|
+
* resolving it, so the owner falls back to its own lease bound rather than being told the fence
|
|
1526
|
+
* succeeded.
|
|
1527
|
+
*/
|
|
1528
|
+
revokeRemoteAdmission(ownerAdmissionId: number): Promise<void> {
|
|
1529
|
+
const localId = this.#remoteByOwnerId?.get(ownerAdmissionId);
|
|
1530
|
+
if (localId === undefined) {
|
|
1531
|
+
// The entry is not installed yet (a revoke that raced ahead of the grant reply): latch the ack
|
|
1532
|
+
// resolver so `#acquireFromOwner` can carry it, and never fence-then-resolve a handle that does
|
|
1533
|
+
// not exist. If the acquire never lands (a revoke for an admission already dropped), `tick`
|
|
1534
|
+
// resolves and sweeps it after a lease so the resolver cannot leak.
|
|
1535
|
+
return new Promise<void>((resolve) => {
|
|
1536
|
+
const map = (this.#pendingRemoteRevokes ??= new Map());
|
|
1537
|
+
const pending = map.get(ownerAdmissionId);
|
|
1538
|
+
if (pending) pending.resolvers.push(resolve);
|
|
1539
|
+
else map.set(ownerAdmissionId, { resolvers: [resolve], at: this.#monotonic() });
|
|
1540
|
+
this.#startTicking();
|
|
1541
|
+
});
|
|
1542
|
+
}
|
|
1543
|
+
const remote = this.#remoteAdmissions?.get(localId);
|
|
1544
|
+
if (!remote) return Promise.resolve();
|
|
1545
|
+
remote.revoked = true;
|
|
1546
|
+
if (remote.revoke === noRevoke) {
|
|
1547
|
+
// The handle has not registered its revoker yet; resolve when `registerAdmission` fences it.
|
|
1548
|
+
return new Promise<void>((resolve) => remote.fenceWaiters.push(resolve));
|
|
1549
|
+
}
|
|
1550
|
+
return Promise.resolve(this.#fenceRemoteAdmission(localId, remote));
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
/**
|
|
1554
|
+
* Fire a remote admission's revoker and report when the handle is PROVABLY fenced. `registerAdmission`
|
|
1555
|
+
* accepts an async revoker, so an outcome that is promise-like is not a fence until it fulfils: the
|
|
1556
|
+
* entry is dropped — and any ack waiting on it settled — only then. A rejection is not a fence at all,
|
|
1557
|
+
* so it propagates and the entry is LEFT for the lease sweep: a retry can fire the revoker again, and
|
|
1558
|
+
* the owner either sees the failure or waits its own lease bound out rather than writing the release
|
|
1559
|
+
* and admitting a successor over a live writer. Unlike a plain release, which retains the entry
|
|
1560
|
+
* because a staged write is still committable until the lease.
|
|
1561
|
+
*/
|
|
1562
|
+
#fenceRemoteAdmission(localId: number, remote: RemoteAdmission): void | Promise<void> {
|
|
1563
|
+
let fenced: void | Promise<void>;
|
|
1564
|
+
try {
|
|
1565
|
+
fenced = remote.revoke();
|
|
1566
|
+
} catch (error) {
|
|
1567
|
+
// A synchronous throw is not a fence either, and it must not escape past the promise this
|
|
1568
|
+
// function's callers hand to the owner: surface it as a rejection instead.
|
|
1569
|
+
warnOnce('a relayed record lock fence failed; the handle lease still bounds it', error);
|
|
1570
|
+
return Promise.reject(error);
|
|
1571
|
+
}
|
|
1572
|
+
if (!isPromiseLike(fenced)) {
|
|
1573
|
+
this.#dropRemoteAdmission(localId, remote);
|
|
1574
|
+
return;
|
|
1575
|
+
}
|
|
1576
|
+
return fenced.then(
|
|
1577
|
+
() => {
|
|
1578
|
+
this.#dropRemoteAdmission(localId, remote);
|
|
1579
|
+
},
|
|
1580
|
+
(error) => {
|
|
1581
|
+
warnOnce('a relayed record lock fence failed; the handle lease still bounds it', error);
|
|
1582
|
+
throw error;
|
|
1583
|
+
}
|
|
1584
|
+
);
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
/**
|
|
1588
|
+
* Fence every relayed handle this thread holds, fail-closed (harper-pro#852). Called when the owner
|
|
1589
|
+
* worker that granted them is gone (its thread exited, so its delegation table died with it): the
|
|
1590
|
+
* handles are no longer backed by any delegation, so nothing here may keep committing. There is no
|
|
1591
|
+
* owner left to tell, so this only revokes locally and drops the entries; a still-pending ack is
|
|
1592
|
+
* resolved, since the handle can no longer commit.
|
|
1593
|
+
*/
|
|
1594
|
+
fenceAllRemoteAdmissions(): void {
|
|
1595
|
+
// Before anything is torn down, so an acquire already in flight to the departed owner fails its
|
|
1596
|
+
// post-reply generation check rather than installing an orphaned admission.
|
|
1597
|
+
this.#relayGeneration++;
|
|
1598
|
+
if (this.#remoteAdmissions) {
|
|
1599
|
+
for (const remote of this.#remoteAdmissions.values()) {
|
|
1600
|
+
fireRevokeAndForget(remote.revoke);
|
|
1601
|
+
this.#settleFenceWaiters(remote);
|
|
1602
|
+
}
|
|
1603
|
+
this.#remoteAdmissions = undefined;
|
|
1604
|
+
this.#remoteByOwnerId = undefined;
|
|
1605
|
+
}
|
|
1606
|
+
if (this.#pendingRemoteRevokes) {
|
|
1607
|
+
for (const entry of this.#pendingRemoteRevokes.values()) for (const resolve of entry.resolvers) resolve();
|
|
1608
|
+
this.#pendingRemoteRevokes = undefined;
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
/**
|
|
1613
|
+
* Forward a release to the owner, naming the admission by the OWNER's id. Core deliberately carries no
|
|
1614
|
+
* owner epoch of its own: a release for an id a replacement owner has since reused is rejected by the
|
|
1615
|
+
* transport, which stamps every release with the owner session captured when the admission was minted
|
|
1616
|
+
* and drops its cached sessions when the coordinating thread changes. An owner ignores anything not
|
|
1617
|
+
* stamped with its own session, so a stale release cannot address a live admission that reuses the id.
|
|
1618
|
+
*/
|
|
1619
|
+
#releaseOnOwnerSafely(key: any, ownerAdmissionId: number): void {
|
|
1620
|
+
// The common relayed unlock is a synchronous post to the owner thread, so it must not cost a
|
|
1621
|
+
// promise chain and two microtasks per unlock: call it directly and only attach a rejection
|
|
1622
|
+
// handler when the transport actually returned a promise.
|
|
1623
|
+
try {
|
|
1624
|
+
const outcome = this.transport.releaseOnOwner?.(this.database, this.table, key, ownerAdmissionId);
|
|
1625
|
+
if (isPromiseLike(outcome))
|
|
1626
|
+
outcome.catch((error) => warnOnce('failed to forward a relayed record lock release to the owner', error));
|
|
1627
|
+
} catch (error) {
|
|
1628
|
+
warnOnce('failed to forward a relayed record lock release to the owner', error);
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/** Resolve every ack waiting on a remote admission's fence, then forget them. */
|
|
1633
|
+
#settleFenceWaiters(remote: RemoteAdmission): void {
|
|
1634
|
+
if (remote.fenceWaiters.length === 0) return;
|
|
1635
|
+
const waiters = remote.fenceWaiters;
|
|
1636
|
+
remote.fenceWaiters = [];
|
|
1637
|
+
for (const resolve of waiters) resolve();
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
/**
|
|
1641
|
+
* Drop remote admissions whose handle lease elapsed a grace ago, at most `MAX_EXPIRIES_PER_TICK` per
|
|
1642
|
+
* tick like the delegation and grant sweeps — a burst of tens of thousands of same-lease admissions
|
|
1643
|
+
* expiring together drains over several ticks rather than deleting them all (and firing a release
|
|
1644
|
+
* message apiece) in one 100 ms turn. The scan skips still-live entries so different lease lengths are
|
|
1645
|
+
* handled. `REMOTE_PRUNE_GRACE_MS` past the lease is what lets the handle's OWN lease timer fire first
|
|
1646
|
+
* and forward its release (`release`) rather than racing this sweep.
|
|
1647
|
+
*
|
|
1648
|
+
* Entries EXAMINED are bounded too, not just deletions: a thread holding many live relayed admissions
|
|
1649
|
+
* would otherwise walk every one of them on every 100 ms tick, purely to find nothing. What is left
|
|
1650
|
+
* unexamined is reached on a later tick, which is safe because this sweep only reclaims memory — an
|
|
1651
|
+
* admission's handle is fenced by its own lease, never by this loop running promptly.
|
|
1652
|
+
*
|
|
1653
|
+
* The scan restarts at the map head each tick rather than carrying a cursor, so above the examine cap
|
|
1654
|
+
* a long-lived prefix would delay collecting expired entries behind it. Accepted deliberately: the cap
|
|
1655
|
+
* is far above any realistic count of concurrent off-owner locks on one thread, and a cursor would
|
|
1656
|
+
* have to walk the prefix anyway to find its resume point, spending the work it meant to save. It
|
|
1657
|
+
* costs memory held longer in a case that should not arise, never correctness.
|
|
1658
|
+
*/
|
|
1659
|
+
#pruneRemoteAdmissions(now: number): void {
|
|
1660
|
+
const remotes = this.#remoteAdmissions;
|
|
1661
|
+
if (!remotes) return;
|
|
1662
|
+
const horizon = now - REMOTE_PRUNE_GRACE_MS;
|
|
1663
|
+
let budget = MAX_EXPIRIES_PER_TICK;
|
|
1664
|
+
let examined = MAX_EXAMINED_PER_TICK;
|
|
1665
|
+
for (const [localId, remote] of remotes) {
|
|
1666
|
+
if (budget <= 0 || examined <= 0) break;
|
|
1667
|
+
examined--;
|
|
1668
|
+
if (remote.expiresMono > horizon) continue;
|
|
1669
|
+
budget--;
|
|
1670
|
+
this.#dropRemoteAdmission(localId, remote, true);
|
|
1671
|
+
}
|
|
1672
|
+
if (remotes.size === 0) {
|
|
1673
|
+
this.#remoteAdmissions = undefined;
|
|
1674
|
+
this.#remoteByOwnerId = undefined;
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
/**
|
|
1679
|
+
* Drop a remote admission entry. `forwardRelease` tells the owner to give the delegation up now —
|
|
1680
|
+
* set when pruning at lease, since an event-loop stall can let this sweep run before the handle's
|
|
1681
|
+
* own lease timer forwards the release, and the owner would otherwise hold `holding` until its own
|
|
1682
|
+
* delegation lease. The forward is idempotent on the owner, so a race with the handle's own release
|
|
1683
|
+
* is harmless. Not set for a fence-drop: the owner is already surrendering that delegation.
|
|
1684
|
+
*/
|
|
1685
|
+
#dropRemoteAdmission(localId: number, remote: RemoteAdmission, forwardRelease = false): void {
|
|
1686
|
+
// Optional: an asynchronous fence can settle after `fenceAllRemoteAdmissions`, `close` or a
|
|
1687
|
+
// handoff has already cleared the map, and a TypeError here would reject an ack whose handle is
|
|
1688
|
+
// in fact fenced, making the owner wait out the lease for nothing.
|
|
1689
|
+
this.#remoteAdmissions?.delete(localId);
|
|
1690
|
+
this.#remoteByOwnerId?.delete(remote.ownerAdmissionId);
|
|
1691
|
+
// The handle's own lease has elapsed, so it fences itself; a still-pending ack may resolve.
|
|
1692
|
+
this.#settleFenceWaiters(remote);
|
|
1693
|
+
if (forwardRelease) this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
/**
|
|
1697
|
+
* Resolve and drop a latched revoke whose admission never installed — a revoke for an admission this
|
|
1698
|
+
* thread had already dropped (fenced or pruned), so the handle is gone and the ack is vacuously
|
|
1699
|
+
* satisfied. Bounded by a lease: past that, no acquire can still be in flight for the id.
|
|
1700
|
+
*/
|
|
1701
|
+
#prunePendingRemoteRevokes(now: number): void {
|
|
1702
|
+
const pending = this.#pendingRemoteRevokes;
|
|
1703
|
+
if (!pending) return;
|
|
1704
|
+
// Bounded like the admission sweep above, and for the same reason: what this tick does not reach,
|
|
1705
|
+
// a later one does. Each waiting ack is independently bounded by the owner's own lease wait.
|
|
1706
|
+
let examined = MAX_EXAMINED_PER_TICK;
|
|
1707
|
+
for (const [ownerId, entry] of pending) {
|
|
1708
|
+
if (examined <= 0) break;
|
|
1709
|
+
examined--;
|
|
1710
|
+
if (now - entry.at < MAX_LOCK_LEASE_MS) continue;
|
|
1711
|
+
pending.delete(ownerId);
|
|
1712
|
+
for (const resolve of entry.resolvers) resolve();
|
|
1713
|
+
}
|
|
1714
|
+
if (pending.size === 0) this.#pendingRemoteRevokes = undefined;
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
/**
|
|
1718
|
+
* End this node's admission for a key. The delegation is deliberately KEPT: that is the
|
|
1719
|
+
* amortization, and the next `lock()` on this node costs nothing. Returns the durable release
|
|
1720
|
+
* write only when the delegation is actually being given up.
|
|
1721
|
+
*/
|
|
1722
|
+
release(key: any, admissionId: number): Promise<void> | void {
|
|
1723
|
+
// A remote admission (harper-pro#852) — the common path once threads.count > 1 — tells the owner
|
|
1724
|
+
// this caller unlocked (its own id, not this thread's local one) but keeps the entry, revoker
|
|
1725
|
+
// included, until the handle's lease runs out: §6 revokes CAPABILITY, not admission, so a write
|
|
1726
|
+
// staged before `unlock()` stays fenceable. Checked before `#keyIdOf` so a relayed unlock does not
|
|
1727
|
+
// pay that key encoding for nothing.
|
|
1728
|
+
const remote = this.#remoteAdmissions?.get(admissionId);
|
|
1729
|
+
if (remote) {
|
|
1730
|
+
this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
|
|
1731
|
+
return undefined;
|
|
1732
|
+
}
|
|
1733
|
+
const keyId = this.#keyIdOf(key);
|
|
1734
|
+
// Addressed by admission, not by key: after a renewal or a replacement the delegation at this
|
|
1735
|
+
// key may not be the one that admitted this handle, and releasing by key alone would either
|
|
1736
|
+
// surrender a successor while its own callers were still inside, or leave this admission on a
|
|
1737
|
+
// delegation nobody can ever drain.
|
|
1738
|
+
const delegation = this.#admissions.get(admissionId);
|
|
1739
|
+
if (!delegation) return undefined;
|
|
1740
|
+
const admission = delegation.admissions.get(admissionId);
|
|
1741
|
+
if (!admission?.holding) return undefined;
|
|
1742
|
+
// The entry stays. Unlocking ends this caller's claim on the DRAIN; the write it staged can
|
|
1743
|
+
// still commit, so the revoker has to remain reachable until the handle's own lease runs out.
|
|
1744
|
+
admission.holding = false;
|
|
1745
|
+
delegation.holding--;
|
|
1746
|
+
if (delegation.holding > 0) return undefined;
|
|
1747
|
+
if (delegation.drained) {
|
|
1748
|
+
// Waking the recall is enough: it surrenders once, on its own path. Surrendering here too
|
|
1749
|
+
// would write a second release entry for the same handoff.
|
|
1750
|
+
const waiters = delegation.drained;
|
|
1751
|
+
delegation.drained = undefined;
|
|
1752
|
+
for (const resolve of waiters) resolve();
|
|
1753
|
+
return undefined;
|
|
1754
|
+
}
|
|
1755
|
+
// Recalled with nobody waiting on the drain — the recall already returned, so this is the path
|
|
1756
|
+
// that gives the delegation up. An un-recalled delegation is deliberately retained.
|
|
1757
|
+
if (delegation.recalled && this.#delegations.get(keyId) === delegation) return this.#surrender(keyId, delegation);
|
|
1758
|
+
return undefined;
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
/**
|
|
1762
|
+
* Apply a control entry from a peer (or a replayed one). Idempotent by `(requester, tsR)`: a
|
|
1763
|
+
* release that arrives twice, or is replayed from the log long after its producer is gone, must be
|
|
1764
|
+
* harmless.
|
|
1765
|
+
*
|
|
1766
|
+
* `author` is the node the entry was actually written by, taken from the audit header rather than
|
|
1767
|
+
* the payload — without it a peer could write a release naming any other node and clear a
|
|
1768
|
+
* delegation it does not hold.
|
|
1769
|
+
*/
|
|
1770
|
+
applyEntry(entry: LockControlEntry, author: string, position?: number): void {
|
|
1771
|
+
// The only boundary peer input crosses into this state machine. A throw here would reach the
|
|
1772
|
+
// replicated apply loop and drop the whole enclosing transaction, so one malformed entry could
|
|
1773
|
+
// stall replication for the database.
|
|
1774
|
+
try {
|
|
1775
|
+
this.#applyEntry(entry, author, position);
|
|
1776
|
+
} catch (error) {
|
|
1777
|
+
warnOnce('failed to apply a record lock control entry', error);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
#applyEntry(entry: LockControlEntry, author: string, position?: number): void {
|
|
1782
|
+
if (this.#closed) return;
|
|
1783
|
+
if (entry.type !== 'lockRelease' || !isNodeName(author)) return;
|
|
1784
|
+
if (entry.requester !== author) return;
|
|
1785
|
+
if (!this.transport.ownsCoordination()) {
|
|
1786
|
+
this.#droppedOffOwner++;
|
|
1787
|
+
const now = this.#monotonic();
|
|
1788
|
+
if (now - this.#lastOffOwnerWarn > WARN_INTERVAL_MS) {
|
|
1789
|
+
this.#lastOffOwnerWarn = now;
|
|
1790
|
+
harperLogger.warn?.('record lock control entries are reaching a non-coordinating thread', {
|
|
1791
|
+
database: this.database,
|
|
1792
|
+
table: this.table,
|
|
1793
|
+
dropped: this.#droppedOffOwner,
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
return;
|
|
1797
|
+
}
|
|
1798
|
+
if (!isFencingToken(entry.token)) return;
|
|
1799
|
+
const keyId = this.#keyIdOf(entry.key);
|
|
1800
|
+
const grant = this.#grants.get(keyId);
|
|
1801
|
+
// Only the delegate named in the live grant can clear it, and only for the exact token it was
|
|
1802
|
+
// issued — generation and incarnation included, since counters restart. A delayed release from a
|
|
1803
|
+
// previous delegation must not clear its successor's.
|
|
1804
|
+
if (!grant || grant.delegate !== author || compareTokens(grant.token, entry.token) !== 0) return;
|
|
1805
|
+
if (entry.dependencies !== null) {
|
|
1806
|
+
const homeMap = this.transport.homeMap(this.database);
|
|
1807
|
+
const matchingHomes = homeMap?.generation === entry.token[0] ? homeMap.homes : undefined;
|
|
1808
|
+
const inherited = matchingHomes ? normalizeDependencies(entry.dependencies, matchingHomes) : undefined;
|
|
1809
|
+
if (
|
|
1810
|
+
inherited &&
|
|
1811
|
+
matchingHomes.includes(author) &&
|
|
1812
|
+
typeof position === 'number' &&
|
|
1813
|
+
Number.isFinite(position) &&
|
|
1814
|
+
position >= 0
|
|
1815
|
+
) {
|
|
1816
|
+
const merged = new Map<string, number>(this.#dependencySets.get(keyId));
|
|
1817
|
+
for (const [origin, dependencyPosition] of inherited)
|
|
1818
|
+
merged.set(origin, Math.max(merged.get(origin) ?? -Infinity, dependencyPosition));
|
|
1819
|
+
merged.set(author, Math.max(merged.get(author) ?? -Infinity, position));
|
|
1820
|
+
if (merged.size <= MAX_LOCK_DEPENDENCIES)
|
|
1821
|
+
this.#rememberDependencies(
|
|
1822
|
+
keyId,
|
|
1823
|
+
[...merged].sort(([a], [b]) => a.localeCompare(b))
|
|
1824
|
+
);
|
|
1825
|
+
else this.#dependencySets.delete(keyId);
|
|
1826
|
+
} else {
|
|
1827
|
+
this.#dependencySets.delete(keyId);
|
|
1828
|
+
}
|
|
1829
|
+
} else if (grant.renewed) this.#dependencySets.delete(keyId);
|
|
1830
|
+
this.#clearGrant(keyId, grant);
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
/**
|
|
1834
|
+
* Inbound delegation request from a peer, for a key this node homes. The home is the single
|
|
1835
|
+
* arbiter, so this is the whole of the exclusion argument: one live grant per key, and a successor
|
|
1836
|
+
* only after the predecessor is recalled-and-drained or provably expired here.
|
|
1837
|
+
*/
|
|
1838
|
+
async onDelegationRequest(request: DelegationRequest): Promise<DelegationReply> {
|
|
1839
|
+
if (this.#closed || !this.transport.ownsCoordination()) return { granted: false, reason: 'not-home' };
|
|
1840
|
+
if (!isNodeName(request.requester) || !isEncodableKey(request.key)) return { granted: false, reason: 'not-home' };
|
|
1841
|
+
if (!isDuration(request.leaseMs, MIN_LOCK_LEASE_MS, MAX_LOCK_LEASE_MS))
|
|
1842
|
+
return { granted: false, reason: 'not-home' };
|
|
1843
|
+
const homeMap = this.transport.homeMap(this.database);
|
|
1844
|
+
if (!homeMap || !this.#generationIsCurrent(homeMap.generation)) return { granted: false, reason: 'generation' };
|
|
1845
|
+
if (homeMap.generation !== request.generation)
|
|
1846
|
+
return { granted: false, reason: 'generation', generation: homeMap.generation };
|
|
1847
|
+
// Membership before state: a node the current map does not name has no claim on a key, and an
|
|
1848
|
+
// authenticated replication identity outlives membership. Without this a decommissioned node
|
|
1849
|
+
// takes delegations against live homes and recalls the legitimate delegate to get them. This is
|
|
1850
|
+
// why `homes` has to name every node that takes a cluster lock, not only the arbiters — see
|
|
1851
|
+
// `LockHomeMap.homes`.
|
|
1852
|
+
if (!homeMap.homes.includes(request.requester)) return { granted: false, reason: 'unknown-node' };
|
|
1853
|
+
// Both sides must agree we are the home, or two arbiters could issue for one key.
|
|
1854
|
+
const keyId = this.#keyIdOf(request.key);
|
|
1855
|
+
if (homeFor(this.#ringKey(keyId), homeMap.homes) !== this.nodeId) return { granted: false, reason: 'not-home' };
|
|
1856
|
+
return this.#grant(keyId, request.key, homeMap, request.leaseMs, request.requester);
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
/**
|
|
1860
|
+
* Inbound recall from a key's home. Revokes capability rather than closing the door: no new
|
|
1861
|
+
* admission may start, live ones are drained, and the release is written only once the delegation
|
|
1862
|
+
* can no longer admit or commit. A recall for a token we no longer hold is a no-op, not an error.
|
|
1863
|
+
*/
|
|
1864
|
+
async onDelegationRecall(recall: DelegationRecall): Promise<void> {
|
|
1865
|
+
// A close that was not a handoff expired and revoked every delegation first, so nothing here can
|
|
1866
|
+
// admit on any token and resolving is honest. A handoff does not reach this: the table's getter
|
|
1867
|
+
// answers the successor, which adopted them.
|
|
1868
|
+
if (this.#closed) return;
|
|
1869
|
+
// Ownership-gated like `onDelegationRequest`, and for the stronger reason: `acquire` refuses off
|
|
1870
|
+
// the owner thread, so a delegation only ever lives on the coordinating one. A recall routed to
|
|
1871
|
+
// any other thread finds no delegation and would resolve — which the home reads as a drained
|
|
1872
|
+
// delegate — while the real delegate keeps admitting. Resolving has to mean nothing on THIS NODE
|
|
1873
|
+
// can admit on that token, and only the owner can say so.
|
|
1874
|
+
if (!this.transport.ownsCoordination())
|
|
1875
|
+
throw new Error('Cluster record lock coordination is not owned by this worker thread');
|
|
1876
|
+
const keyId = this.#keyIdOf(recall.key);
|
|
1877
|
+
const pending = this.#pendingDelegations.get(keyId);
|
|
1878
|
+
if (pending && compareTokens(pending.token, recall.token) === 0) {
|
|
1879
|
+
pending.markRecalled();
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
const delegation = this.#delegations.get(keyId);
|
|
1883
|
+
if (!delegation || compareTokens(delegation.token, recall.token) !== 0) {
|
|
1884
|
+
// A recall can beat the grant reply. Remember its token so that reply cannot install authority
|
|
1885
|
+
// the home already believes drained.
|
|
1886
|
+
const request = this.#pendingRequests.get(keyId);
|
|
1887
|
+
if (request) request.recalledToken = recall.token;
|
|
1888
|
+
return;
|
|
1889
|
+
}
|
|
1890
|
+
delegation.recalled = true;
|
|
1891
|
+
if (delegation.holding > 0) {
|
|
1892
|
+
await new Promise<void>((resolve) => {
|
|
1893
|
+
// A holder that never releases must not hold the recall open past its own lease: the
|
|
1894
|
+
// delegation expires here on our clock, and the home outwaits that by skew anyway. The
|
|
1895
|
+
// timer is dropped by whichever side wins, so a delegation that drains promptly does not
|
|
1896
|
+
// retain one for the rest of its lease.
|
|
1897
|
+
const remaining = Math.max(0, delegation.expiresMono - this.#monotonic());
|
|
1898
|
+
const timer = setTimeout(resolve, remaining);
|
|
1899
|
+
timer.unref?.();
|
|
1900
|
+
(delegation.drained ||= []).push(() => {
|
|
1901
|
+
clearTimeout(timer);
|
|
1902
|
+
resolve();
|
|
1903
|
+
});
|
|
1904
|
+
});
|
|
1905
|
+
}
|
|
1906
|
+
// The swap may have landed during the drain. The delegation OBJECT moved to the successor, so
|
|
1907
|
+
// surrendering here would revoke through an index this coordinator no longer owns and leave the
|
|
1908
|
+
// successor still holding a delegation whose release has already been written.
|
|
1909
|
+
await this.#authority().#surrender(keyId, delegation);
|
|
1910
|
+
}
|
|
1911
|
+
|
|
1912
|
+
/** Expire delegations and grants whose deadlines have passed. Bounded work per call. */
|
|
1913
|
+
tick(): void {
|
|
1914
|
+
const now = this.#monotonic();
|
|
1915
|
+
// A second, weaker signal than the incarnation: it catches a thread that stopped coordinating
|
|
1916
|
+
// without anything else starting, which advances no incarnation. Rate-limited well below the
|
|
1917
|
+
// tick because it cannot matter faster than a delegation lease, and it costs nothing at all on a
|
|
1918
|
+
// coordinator holding nothing — that one has already stopped ticking.
|
|
1919
|
+
if (now - this.#lastOwnershipPollMono >= OWNERSHIP_POLL_MS) {
|
|
1920
|
+
this.#lastOwnershipPollMono = now;
|
|
1921
|
+
this.#observeOwnership();
|
|
1922
|
+
}
|
|
1923
|
+
this.#pruneRemoteAdmissions(now);
|
|
1924
|
+
this.#prunePendingRemoteRevokes(now);
|
|
1925
|
+
// The budget counts EXPIRIES, not entries examined. Spending it on live entries would let a
|
|
1926
|
+
// table with more than a budget's worth of continuously renewed grants starve every expired
|
|
1927
|
+
// one behind them, and an uncollected expired grant answers `contended` to every other node.
|
|
1928
|
+
let budget = MAX_EXPIRIES_PER_TICK;
|
|
1929
|
+
for (const [keyId, delegation] of this.#delegations) {
|
|
1930
|
+
if (budget <= 0) break;
|
|
1931
|
+
if (delegation.expiresMono > now) continue;
|
|
1932
|
+
budget--;
|
|
1933
|
+
// A delegate may drop early; the handles it admitted are fenced by their own lease, which
|
|
1934
|
+
// never outlives the delegation that admitted them.
|
|
1935
|
+
this.#delegations.delete(keyId);
|
|
1936
|
+
this.#revokeAll(delegation);
|
|
1937
|
+
const waiters = delegation.drained;
|
|
1938
|
+
if (waiters) {
|
|
1939
|
+
delegation.drained = undefined;
|
|
1940
|
+
for (const resolve of waiters) resolve();
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
budget = MAX_EXPIRIES_PER_TICK;
|
|
1944
|
+
for (const [keyId, grant] of this.#grants) {
|
|
1945
|
+
if (budget <= 0) break;
|
|
1946
|
+
// A home may NEVER forget a grant before its expiry — that asymmetry is the safety rule.
|
|
1947
|
+
if (grant.expiresMono > now) continue;
|
|
1948
|
+
budget--;
|
|
1949
|
+
this.#expireGrant(keyId, grant);
|
|
1950
|
+
}
|
|
1951
|
+
if (
|
|
1952
|
+
this.#delegations.size === 0 &&
|
|
1953
|
+
this.#grants.size === 0 &&
|
|
1954
|
+
!this.#remoteAdmissions &&
|
|
1955
|
+
!this.#pendingRemoteRevokes
|
|
1956
|
+
)
|
|
1957
|
+
tickingCoordinators.delete(this);
|
|
1958
|
+
}
|
|
1959
|
+
|
|
1960
|
+
/**
|
|
1961
|
+
* One table's half of `quiesceDelegations`. Never throws for a single grant: a transition needs to
|
|
1962
|
+
* know exactly what is still live, and one unreachable delegate must not hide the rest.
|
|
1963
|
+
*/
|
|
1964
|
+
async quiesce(result: QuiesceResult, remaining: () => number): Promise<void> {
|
|
1965
|
+
if (this.#closed) return;
|
|
1966
|
+
// Delegate side first: this is what admits, and surrendering is purely local — it cannot be
|
|
1967
|
+
// refused by an unreachable peer, so it succeeds even when the recalls below do not.
|
|
1968
|
+
for (const delegation of [...this.#delegations.values()]) {
|
|
1969
|
+
try {
|
|
1970
|
+
await withDeadline(this.onDelegationRecall({ key: delegation.key, token: delegation.token }), remaining());
|
|
1971
|
+
result.surrendered++;
|
|
1972
|
+
} catch (error) {
|
|
1973
|
+
result.outstanding.push({
|
|
1974
|
+
table: this.table,
|
|
1975
|
+
key: delegation.key,
|
|
1976
|
+
reason: `this node still holds a delegation it could not drain: ${(error as Error)?.message ?? error}`,
|
|
1977
|
+
});
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
// Home side: tell delegates elsewhere to stop. `#beginRecall` owns the retry and confirmation
|
|
1981
|
+
// bookkeeping; this only drives it and reports what it did not confirm.
|
|
1982
|
+
for (const [keyId, grant] of [...this.#grants]) {
|
|
1983
|
+
if (grant.recallConfirmed) {
|
|
1984
|
+
result.recalled++;
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
try {
|
|
1988
|
+
this.#beginRecall(keyId, grant);
|
|
1989
|
+
if (grant.recalling) await withDeadline(grant.recalling, remaining());
|
|
1990
|
+
if (grant.recallConfirmed) result.recalled++;
|
|
1991
|
+
else
|
|
1992
|
+
result.outstanding.push({
|
|
1993
|
+
table: this.table,
|
|
1994
|
+
key: grant.key,
|
|
1995
|
+
delegate: grant.delegate,
|
|
1996
|
+
reason: 'the delegate did not confirm it stopped admitting',
|
|
1997
|
+
});
|
|
1998
|
+
} catch (error) {
|
|
1999
|
+
result.outstanding.push({
|
|
2000
|
+
table: this.table,
|
|
2001
|
+
key: grant.key,
|
|
2002
|
+
delegate: grant.delegate,
|
|
2003
|
+
reason: `recall failed: ${(error as Error)?.message ?? error}`,
|
|
2004
|
+
});
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
/**
|
|
2010
|
+
* Stop this coordinator and invalidate what it issued. Expiring every delegation is the part that
|
|
2011
|
+
* matters: `close()` runs when a transport is replaced (a component reload is enough), and a
|
|
2012
|
+
* successor coordinator must not be able to grant a key whose predecessor handles are still live.
|
|
2013
|
+
*/
|
|
2014
|
+
close(): void {
|
|
2015
|
+
this.#closed = true;
|
|
2016
|
+
liveCoordinators.delete(this);
|
|
2017
|
+
// State that was handed to a successor is that coordinator's now; expiring it here would
|
|
2018
|
+
// invalidate delegations the successor is correctly still honouring.
|
|
2019
|
+
if (this.#handedOff) {
|
|
2020
|
+
tickingCoordinators.delete(this);
|
|
2021
|
+
return;
|
|
2022
|
+
}
|
|
2023
|
+
for (const delegation of this.#delegations.values()) {
|
|
2024
|
+
delegation.recalled = true;
|
|
2025
|
+
delegation.expiresMono = -Infinity;
|
|
2026
|
+
this.#revokeAll(delegation);
|
|
2027
|
+
const waiters = delegation.drained;
|
|
2028
|
+
if (waiters) {
|
|
2029
|
+
delegation.drained = undefined;
|
|
2030
|
+
for (const resolve of waiters) resolve();
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
for (const pending of this.#pendingDelegations.values()) pending.markRecalled();
|
|
2034
|
+
// Remote admissions (harper-pro#852) that were NOT handed to a successor: this thread is going
|
|
2035
|
+
// away, so fence their handles (fail closed) and tell the owner it may give the delegation up now
|
|
2036
|
+
// rather than hold it for a full lease waiting on a worker that has gone.
|
|
2037
|
+
if (this.#remoteAdmissions) {
|
|
2038
|
+
for (const remote of this.#remoteAdmissions.values()) {
|
|
2039
|
+
fireRevokeAndForget(remote.revoke);
|
|
2040
|
+
this.#settleFenceWaiters(remote);
|
|
2041
|
+
this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
|
|
2042
|
+
}
|
|
2043
|
+
this.#remoteAdmissions = undefined;
|
|
2044
|
+
this.#remoteByOwnerId = undefined;
|
|
2045
|
+
}
|
|
2046
|
+
// Any ack still waiting on an entry that never installed cannot be honored here; resolve it so the
|
|
2047
|
+
// owner's revoke wait does not hang on a thread that is gone (its own lease bounds it regardless).
|
|
2048
|
+
if (this.#pendingRemoteRevokes) {
|
|
2049
|
+
for (const entry of this.#pendingRemoteRevokes.values()) for (const resolve of entry.resolvers) resolve();
|
|
2050
|
+
this.#pendingRemoteRevokes = undefined;
|
|
2051
|
+
}
|
|
2052
|
+
// The delegations THIS node issued as a home outlive it: no peer sees a local close, so each one
|
|
2053
|
+
// stands until its own deadline. Leave the latest of those deadlines, and the counter, for
|
|
2054
|
+
// whatever coordinator takes this table next.
|
|
2055
|
+
let grantableAfterMono = -Infinity;
|
|
2056
|
+
for (const grant of this.#grants.values()) {
|
|
2057
|
+
// A grant to THIS node is settled by the same close: the loop above revoked the delegation it
|
|
2058
|
+
// authorized. Only what another node holds outlives this coordinator unseen.
|
|
2059
|
+
if (grant.delegate === this.nodeId) continue;
|
|
2060
|
+
if (grant.expiresMono > grantableAfterMono) grantableAfterMono = grant.expiresMono;
|
|
2061
|
+
}
|
|
2062
|
+
const retirementKey = this.#retirementKey();
|
|
2063
|
+
const retired = retiredCoordinators.get(retirementKey);
|
|
2064
|
+
retiredCoordinators.set(retirementKey, {
|
|
2065
|
+
grantableAfterMono: Math.max(retired?.grantableAfterMono ?? -Infinity, grantableAfterMono),
|
|
2066
|
+
counter: Math.max(retired?.counter ?? 0, this.#counter),
|
|
2067
|
+
});
|
|
2068
|
+
this.#delegations.clear();
|
|
2069
|
+
this.#pendingDelegations.clear();
|
|
2070
|
+
this.#pendingRequests.clear();
|
|
2071
|
+
this.#grants.clear();
|
|
2072
|
+
this.#grantsByRequester.clear();
|
|
2073
|
+
this.#dependencySets.clear();
|
|
2074
|
+
this.#everDelegated.clear();
|
|
2075
|
+
tickingCoordinators.delete(this);
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
/**
|
|
2079
|
+
* Ownership as this thread can actually observe it. Only a LOSS is recorded: regaining it is the
|
|
2080
|
+
* grant path's business, because that is where the incarnation is available to say whether anything
|
|
2081
|
+
* else coordinated in between.
|
|
2082
|
+
*/
|
|
2083
|
+
#observeOwnership(): void {
|
|
2084
|
+
let owns: boolean;
|
|
2085
|
+
try {
|
|
2086
|
+
owns = this.transport.ownsCoordination();
|
|
2087
|
+
} catch {
|
|
2088
|
+
// A transport that cannot answer is not proof this thread kept coordinating, and the whole
|
|
2089
|
+
// point of the horizon is what happened while it did not.
|
|
2090
|
+
owns = false;
|
|
2091
|
+
}
|
|
2092
|
+
if (!owns) this.#ownedSinceMono = undefined;
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
/**
|
|
2096
|
+
* The horizon owning coordination imposes, anchored at the instant this coordinator STARTED
|
|
2097
|
+
* coordinating under the incarnation it is granting under — see `#ownedSinceMono`.
|
|
2098
|
+
*
|
|
2099
|
+
* `homeIncarnation` is the signal, not `ownsCoordination()`. Sampling a boolean proves the answer
|
|
2100
|
+
* at the instant it is read and never that ownership was unbroken between two reads, so ownership
|
|
2101
|
+
* alternating faster than the sample interval could alias away entirely. §5.1 makes the incarnation
|
|
2102
|
+
* advance once per COORDINATION incarnation — that is what keeps the fencing token orderable — so a
|
|
2103
|
+
* value this coordinator has not granted under is the transport stating that something else
|
|
2104
|
+
* coordinated for this node, whatever the boolean said in between.
|
|
2105
|
+
*/
|
|
2106
|
+
#ownershipHorizon(now: number, homeIncarnation: number): number {
|
|
2107
|
+
if (this.#coordinatingIncarnation !== homeIncarnation || this.#ownedSinceMono === undefined) {
|
|
2108
|
+
this.#loseFreshnessHistory();
|
|
2109
|
+
// Ownership STARTED here, which is the one thing the waiver cannot cover.
|
|
2110
|
+
// `grantableAfterMono` attests that no previous INCARNATION OF THIS PROCESS had issued
|
|
2111
|
+
// delegations — a cold start, a fresh database, a test — and says nothing about the sibling
|
|
2112
|
+
// thread that was coordinating until this instant. The constructor records both directly, so
|
|
2113
|
+
// a coordinator that has coordinated since it was built never reaches this and keeps the
|
|
2114
|
+
// waiver; every other way of arriving at ownership lands here and loses it.
|
|
2115
|
+
this.#coordinatingIncarnation = homeIncarnation;
|
|
2116
|
+
this.#quarantineWaived = false;
|
|
2117
|
+
this.#ownedSinceMono = now;
|
|
2118
|
+
}
|
|
2119
|
+
if (this.#quarantineWaived) return -Infinity;
|
|
2120
|
+
return this.#ownedSinceMono + DELEGATION_LEASE_MS + this.#skewMs;
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
/** Fail closed on a generation older than one already ACTED on here (see `highestGeneration`). */
|
|
2124
|
+
#generationIsCurrent(generation: number): boolean {
|
|
2125
|
+
const highest = highestGeneration.get(this.database);
|
|
2126
|
+
return highest === undefined || generation >= highest;
|
|
2127
|
+
}
|
|
2128
|
+
|
|
2129
|
+
/**
|
|
2130
|
+
* Raise the floor, at the moment authority is actually taken under this generation — a token
|
|
2131
|
+
* minted or a delegation installed — and never merely on reading a map.
|
|
2132
|
+
*
|
|
2133
|
+
* Observing was the obvious place and it is the wrong one: a single `homeMap()` that returns a
|
|
2134
|
+
* too-large generation once, from a partial publish or a transport glitch, would pin the floor
|
|
2135
|
+
* above anything the operator ever publishes and fail every later lock on this database until the
|
|
2136
|
+
* thread restarts. Nothing was minted under that reading, so nothing needs protecting from it. The
|
|
2137
|
+
* invariant only ever needed to be "never mint below a generation already minted".
|
|
2138
|
+
*/
|
|
2139
|
+
#recordGenerationActedOn(generation: number): void {
|
|
2140
|
+
const highest = highestGeneration.get(this.database);
|
|
2141
|
+
if (highest === undefined || generation > highest) highestGeneration.set(this.database, generation);
|
|
2142
|
+
}
|
|
2143
|
+
|
|
2144
|
+
#prepareFreshnessGeneration(generation: number): void {
|
|
2145
|
+
if (this.#freshnessGeneration === undefined) {
|
|
2146
|
+
this.#freshnessGeneration = generation;
|
|
2147
|
+
return;
|
|
2148
|
+
}
|
|
2149
|
+
if (this.#freshnessGeneration === generation) return;
|
|
2150
|
+
this.#freshnessGeneration = generation;
|
|
2151
|
+
this.#loseFreshnessHistory();
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
#loseFreshnessHistory(): void {
|
|
2155
|
+
this.#dependencySets.clear();
|
|
2156
|
+
this.#everDelegated.clear();
|
|
2157
|
+
this.#trustVirginKeys = false;
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
#freshnessFor(keyId: unknown): LockDependencySet | null {
|
|
2161
|
+
const retained = this.#dependencySets.get(keyId);
|
|
2162
|
+
if (retained) {
|
|
2163
|
+
this.#dependencySets.delete(keyId);
|
|
2164
|
+
this.#dependencySets.set(keyId, retained);
|
|
2165
|
+
return retained;
|
|
2166
|
+
}
|
|
2167
|
+
return this.#trustVirginKeys && !this.#everDelegated.has(keyId) ? [] : null;
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
#rememberDependencies(keyId: unknown, dependencies: LockDependencySet): void {
|
|
2171
|
+
this.#dependencySets.delete(keyId);
|
|
2172
|
+
this.#dependencySets.set(keyId, dependencies);
|
|
2173
|
+
while (this.#dependencySets.size > MAX_DEPENDENCY_SETS_PER_TABLE)
|
|
2174
|
+
this.#dependencySets.delete(this.#dependencySets.keys().next().value);
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
#retirementKey(): string {
|
|
2178
|
+
return `${this.database}\u0000${this.table}`;
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
#ringKey(keyId: unknown): string {
|
|
2182
|
+
return ringKeyFor(this.database, this.table, keyId);
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
/** The coordinator holding this node's authority now: this one, or the end of the handoff chain. */
|
|
2186
|
+
#authority(): LockCoordinator {
|
|
2187
|
+
let coordinator: LockCoordinator = this;
|
|
2188
|
+
while (coordinator.#successor) coordinator = coordinator.#successor;
|
|
2189
|
+
return coordinator;
|
|
2190
|
+
}
|
|
2191
|
+
|
|
2192
|
+
#liveDelegation(keyId: unknown, leaseMs: number, generation: number): Delegation | undefined {
|
|
2193
|
+
const delegation = this.#delegations.get(keyId);
|
|
2194
|
+
if (!delegation || delegation.recalled) return undefined;
|
|
2195
|
+
// A delegation is authority within ONE generation. After a generation change the key may have
|
|
2196
|
+
// been re-homed, and the new home knows nothing of this token — so keeping it would let this
|
|
2197
|
+
// node admit alongside whoever the new home grants.
|
|
2198
|
+
if (delegation.token[0] !== generation) {
|
|
2199
|
+
// Authority is gone, not merely stale: the key may have been re-homed to a node that knows
|
|
2200
|
+
// nothing of this token. Forgetting the delegation without revoking would leave its handles
|
|
2201
|
+
// able to commit alongside whatever the new home grants.
|
|
2202
|
+
this.#delegations.delete(keyId);
|
|
2203
|
+
this.#revokeAll(delegation);
|
|
2204
|
+
return undefined;
|
|
2205
|
+
}
|
|
2206
|
+
// The admission may not outlive the delegation that admitted it, so a delegation without room
|
|
2207
|
+
// for the whole lease is renewed rather than stretched.
|
|
2208
|
+
if (delegation.expiresMono - this.#monotonic() < leaseMs) return undefined;
|
|
2209
|
+
return delegation;
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
#admit(delegation: Delegation, leaseMs: number): LockRound {
|
|
2213
|
+
const mintedMono = this.#monotonic();
|
|
2214
|
+
// Admissions that unlocked are kept only until their handle's own lease fences it. Collecting
|
|
2215
|
+
// them here rather than on a timer keeps the work on the path that creates it, and a delegation
|
|
2216
|
+
// can accumulate at most one entry per overlapping lock in its own lease.
|
|
2217
|
+
this.#pruneAdmissions(delegation, mintedMono);
|
|
2218
|
+
const admissionId = this.#nextAdmissionId++;
|
|
2219
|
+
// The entry exists from the instant of admission, so a recall between admit and register still
|
|
2220
|
+
// sees it.
|
|
2221
|
+
delegation.admissions.set(admissionId, { revoke: noRevoke, expiresMono: mintedMono + leaseMs, holding: true });
|
|
2222
|
+
delegation.holding++;
|
|
2223
|
+
this.#admissions.set(admissionId, delegation);
|
|
2224
|
+
return { tsR: this.#nextTimestamp(), mintedMono, admissionId };
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
/**
|
|
2228
|
+
* Drop admissions whose handle's lease has run out; they can no longer commit anything.
|
|
2229
|
+
*
|
|
2230
|
+
* Insertion order is monotonic order, not expiry order: under one lease length the leading run IS
|
|
2231
|
+
* the expired set, but a longer-lease admission at the head hides every shorter one behind it for
|
|
2232
|
+
* its own remaining lease. Scanning the whole map on every admission instead would be quadratic on
|
|
2233
|
+
* the hot key this design exists to make cheap, so it is swept only once it has outgrown the live
|
|
2234
|
+
* set the previous sweep measured — amortized O(1) per admission, and the map stays within twice
|
|
2235
|
+
* the live set rather than growing at the lock rate.
|
|
2236
|
+
*
|
|
2237
|
+
* Never has a drain to wake: `#admit` reaches neither call site for a recalled delegation.
|
|
2238
|
+
*/
|
|
2239
|
+
#pruneAdmissions(delegation: Delegation, now: number): void {
|
|
2240
|
+
for (const [admissionId, admission] of delegation.admissions) {
|
|
2241
|
+
if (admission.expiresMono > now) break;
|
|
2242
|
+
this.#dropAdmission(delegation, admissionId, admission);
|
|
2243
|
+
}
|
|
2244
|
+
if (delegation.admissions.size < delegation.sweepAtSize) return;
|
|
2245
|
+
for (const [admissionId, admission] of delegation.admissions)
|
|
2246
|
+
if (admission.expiresMono <= now) this.#dropAdmission(delegation, admissionId, admission);
|
|
2247
|
+
delegation.sweepAtSize = delegation.admissions.size * 2 + ADMISSION_SWEEP_FLOOR;
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
#dropAdmission(delegation: Delegation, admissionId: number, admission: Admission): void {
|
|
2251
|
+
delegation.admissions.delete(admissionId);
|
|
2252
|
+
this.#admissions.delete(admissionId);
|
|
2253
|
+
if (admission.holding) delegation.holding--;
|
|
2254
|
+
}
|
|
2255
|
+
|
|
2256
|
+
/**
|
|
2257
|
+
* Register how to revoke the handle an admission produced. `Table.lock()` calls this once the
|
|
2258
|
+
* handle has joined the round, so a recall can fence a write that was staged and then unlocked.
|
|
2259
|
+
*/
|
|
2260
|
+
registerAdmission(admissionId: number, revoke: () => void | Promise<void>): void {
|
|
2261
|
+
const admission = this.#admissions.get(admissionId)?.admissions.get(admissionId);
|
|
2262
|
+
if (admission) {
|
|
2263
|
+
admission.revoke = revoke;
|
|
2264
|
+
return;
|
|
2265
|
+
}
|
|
2266
|
+
// A remote admission (harper-pro#852): the delegation lives on the owner, the handle here. A revoke
|
|
2267
|
+
// that already latched (it beat this registration) fences the handle now and releases the ack that
|
|
2268
|
+
// was waiting on the fence; otherwise store the revoker for a later recall.
|
|
2269
|
+
const remote = this.#remoteAdmissions?.get(admissionId);
|
|
2270
|
+
if (remote) {
|
|
2271
|
+
remote.revoke = revoke;
|
|
2272
|
+
if (remote.revoked) {
|
|
2273
|
+
// A revoke latched before this registration: fence now, then drop the entry and release the
|
|
2274
|
+
// ack that was waiting on the fence. `#dropRemoteAdmission` is what resolves that ack, so an
|
|
2275
|
+
// async revoker must settle FIRST — resolving it early would tell the owner this handle is
|
|
2276
|
+
// fenced while it still is not, and the owner would write the release over a live writer. A fence
|
|
2277
|
+
// that FAILS settles nothing: the entry is left for the lease sweep, which resolves the ack once
|
|
2278
|
+
// the handle is provably dead. The failure is warned about inside and has nobody here to go to,
|
|
2279
|
+
// so it is absorbed rather than left to Node's unhandled-rejection policy.
|
|
2280
|
+
const fenced = this.#fenceRemoteAdmission(admissionId, remote);
|
|
2281
|
+
if (isPromiseLike(fenced)) fenced.catch(() => {});
|
|
2282
|
+
}
|
|
2283
|
+
return;
|
|
2284
|
+
}
|
|
2285
|
+
// The admission was already revoked or collected between admit and register — the handle has no
|
|
2286
|
+
// authority to keep, so revoke it now (safely: a relay revoker collected in this window is async
|
|
2287
|
+
// and could reject on a dead port) rather than leaving it unfenced.
|
|
2288
|
+
fireRevokeAndForget(revoke);
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2291
|
+
/**
|
|
2292
|
+
* Install a granted delegation, with its deadline anchored at the moment the request was SENT
|
|
2293
|
+
* rather than at the moment the reply arrived. The home started its own clock when it granted, so
|
|
2294
|
+
* anchoring on arrival would hand a delayed reply more time than the home is holding the key for —
|
|
2295
|
+
* and a reply delayed past the whole delegation must be discarded, not installed (§5.2).
|
|
2296
|
+
*/
|
|
2297
|
+
#installDelegation(
|
|
2298
|
+
keyId: unknown,
|
|
2299
|
+
key: any,
|
|
2300
|
+
token: FencingToken,
|
|
2301
|
+
leaseMs: number,
|
|
2302
|
+
requestedAtMono: number,
|
|
2303
|
+
dependencies: LockDependencySet
|
|
2304
|
+
): Delegation | undefined {
|
|
2305
|
+
const expiresMono = requestedAtMono + leaseMs;
|
|
2306
|
+
if (expiresMono <= this.#monotonic()) return undefined;
|
|
2307
|
+
// The delegate side of taking authority under a generation: installing this makes it admit under
|
|
2308
|
+
// `token[0]`, so that is the floor a later map may not go below.
|
|
2309
|
+
this.#recordGenerationActedOn(token[0]);
|
|
2310
|
+
const existing = this.#delegations.get(keyId);
|
|
2311
|
+
// A reply that lost a race with a newer delegation for the same key must not move it backwards.
|
|
2312
|
+
if (existing && compareTokens(existing.token, token) >= 0) return existing;
|
|
2313
|
+
if (existing && !existing.recalled) {
|
|
2314
|
+
// A RENEWAL of authority this node never lost. Advance the token and the deadline in place so
|
|
2315
|
+
// the admissions it is still answerable for ride along; installing a fresh object here
|
|
2316
|
+
// orphaned them, and the next recall then surrendered while a live handle could still commit.
|
|
2317
|
+
existing.token = token;
|
|
2318
|
+
existing.expiresMono = expiresMono;
|
|
2319
|
+
this.#startTicking();
|
|
2320
|
+
return existing;
|
|
2321
|
+
}
|
|
2322
|
+
// Replacing a recalled delegation: its handles were revoked at surrender, so nothing carries.
|
|
2323
|
+
if (existing) this.#revokeAll(existing);
|
|
2324
|
+
const delegation: Delegation = {
|
|
2325
|
+
key,
|
|
2326
|
+
token,
|
|
2327
|
+
dependencies,
|
|
2328
|
+
expiresMono,
|
|
2329
|
+
recalled: false,
|
|
2330
|
+
admissions: new Map(),
|
|
2331
|
+
holding: 0,
|
|
2332
|
+
sweepAtSize: ADMISSION_SWEEP_FLOOR,
|
|
2333
|
+
};
|
|
2334
|
+
this.#delegations.set(keyId, delegation);
|
|
2335
|
+
this.#startTicking();
|
|
2336
|
+
return delegation;
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
async #establishFreshness(
|
|
2340
|
+
key: any,
|
|
2341
|
+
requirement: LockDependencySet | null,
|
|
2342
|
+
homes: readonly string[],
|
|
2343
|
+
deadlineMono: number,
|
|
2344
|
+
pending?: PendingDelegation
|
|
2345
|
+
): Promise<LockDependencySet> {
|
|
2346
|
+
if (pending?.recalled) throw PENDING_DELEGATION_RECALLED;
|
|
2347
|
+
if (requirement?.length === 0) return requirement;
|
|
2348
|
+
const remaining = deadlineMono - this.#monotonic();
|
|
2349
|
+
if (remaining <= 0) throw new Error('the lock wait elapsed before its freshness barrier started');
|
|
2350
|
+
const established = Promise.resolve().then(() =>
|
|
2351
|
+
this.transport.establishLockFreshness(this.database, this.table, key, requirement, remaining)
|
|
2352
|
+
);
|
|
2353
|
+
const timeout = delay(remaining);
|
|
2354
|
+
established.then(
|
|
2355
|
+
() => timeout.cancel(),
|
|
2356
|
+
() => timeout.cancel()
|
|
2357
|
+
);
|
|
2358
|
+
const alternatives: Promise<LockDependencySet | void>[] = [
|
|
2359
|
+
established,
|
|
2360
|
+
timeout.promise.then(() => {
|
|
2361
|
+
throw new Error('the lock wait elapsed before its freshness barrier completed');
|
|
2362
|
+
}),
|
|
2363
|
+
];
|
|
2364
|
+
if (pending)
|
|
2365
|
+
alternatives.push(
|
|
2366
|
+
pending.recalledPromise.then(() => {
|
|
2367
|
+
timeout.cancel();
|
|
2368
|
+
throw PENDING_DELEGATION_RECALLED;
|
|
2369
|
+
})
|
|
2370
|
+
);
|
|
2371
|
+
const result = await Promise.race(alternatives);
|
|
2372
|
+
if (requirement !== null) return requirement;
|
|
2373
|
+
const recovered = normalizeDependencies(result);
|
|
2374
|
+
if (!recovered) throw new Error('the recovery barrier returned no usable applied-position set');
|
|
2375
|
+
const allowedOrigins = new Set(homes);
|
|
2376
|
+
return recovered.filter(([origin]) => allowedOrigins.has(origin));
|
|
2377
|
+
}
|
|
2378
|
+
|
|
2379
|
+
/** This node is the key's home: grant to itself through exactly the same table a peer would use. */
|
|
2380
|
+
#grantLocally(keyId: unknown, key: any, homeMap: LockHomeMap, leaseMs: number): DelegationReply {
|
|
2381
|
+
return this.#grant(keyId, key, homeMap, leaseMs, this.nodeId);
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
async #requestRemotely(
|
|
2385
|
+
home: string,
|
|
2386
|
+
keyId: unknown,
|
|
2387
|
+
key: any,
|
|
2388
|
+
homeMap: LockHomeMap,
|
|
2389
|
+
leaseMs: number,
|
|
2390
|
+
deadlineMono: number
|
|
2391
|
+
): Promise<DelegationReply> {
|
|
2392
|
+
try {
|
|
2393
|
+
// A half-open connection to the home would otherwise leave `lock()` pending forever, past
|
|
2394
|
+
// its own timeout and past the native lease — and a reply arriving after that lease has
|
|
2395
|
+
// fired cannot be joined to the handle anyway.
|
|
2396
|
+
const remaining = Math.max(1, deadlineMono - this.#monotonic());
|
|
2397
|
+
let raced = false;
|
|
2398
|
+
const requested = Promise.resolve(
|
|
2399
|
+
this.transport.requestDelegation(home, this.database, this.table, {
|
|
2400
|
+
key,
|
|
2401
|
+
requester: this.nodeId,
|
|
2402
|
+
generation: homeMap.generation,
|
|
2403
|
+
leaseMs,
|
|
2404
|
+
})
|
|
2405
|
+
);
|
|
2406
|
+
// A reply that arrives after we stopped waiting still granted us the key on the home, which
|
|
2407
|
+
// would then hold it for the whole delegation while every other node is denied. Hand it back.
|
|
2408
|
+
// Through the authority, not through this object: a transport swap can land while the reply
|
|
2409
|
+
// is in flight, and `handOffTo` empties this coordinator's delegations. The handback's
|
|
2410
|
+
// "not while a delegation for the key is held" guard would then read an empty map and give
|
|
2411
|
+
// back a grant that still backs the successor's live delegation.
|
|
2412
|
+
requested.then(
|
|
2413
|
+
(reply) => {
|
|
2414
|
+
if (raced && reply?.granted && reply.token) this.#authority().#releaseUnclaimedGrant(key, reply.token);
|
|
2415
|
+
},
|
|
2416
|
+
() => {}
|
|
2417
|
+
);
|
|
2418
|
+
const timeout = delay(remaining);
|
|
2419
|
+
// Dropped once the race settles: a reply that beats the timeout would otherwise leave a timer
|
|
2420
|
+
// holding this closure for the caller's whole remaining wait.
|
|
2421
|
+
requested.then(
|
|
2422
|
+
() => timeout.cancel(),
|
|
2423
|
+
() => timeout.cancel()
|
|
2424
|
+
);
|
|
2425
|
+
return await Promise.race([
|
|
2426
|
+
requested,
|
|
2427
|
+
timeout.promise.then(() => {
|
|
2428
|
+
raced = true;
|
|
2429
|
+
return { granted: false, reason: 'timeout', retryAfterMs: 0 } as DelegationReply;
|
|
2430
|
+
}),
|
|
2431
|
+
]);
|
|
2432
|
+
} catch (error) {
|
|
2433
|
+
// An unreachable home blocks only the keys it homes, which is the availability property the
|
|
2434
|
+
// whole design exists for — it is not a reason to admit without one.
|
|
2435
|
+
throw new LockUnavailableError(
|
|
2436
|
+
`Could not reach ${home}, the home node for this key on ${this.database}.${this.table}: ${(error as Error)?.message ?? error}`
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
|
|
2441
|
+
/**
|
|
2442
|
+
* Give back a delegation this node asked for but stopped waiting on. Without it the home holds the
|
|
2443
|
+
* key for a delegation nobody is using, and every other node is denied for its full duration.
|
|
2444
|
+
*/
|
|
2445
|
+
#releaseUnclaimedGrant(key: any, token: FencingToken): Promise<void> | void {
|
|
2446
|
+
const keyId = this.#keyIdOf(key);
|
|
2447
|
+
const held = this.#delegations.get(keyId);
|
|
2448
|
+
// ANY live delegation for this key means this node is using it, and the tokens need not match.
|
|
2449
|
+
// A duplicate or delayed request from this node renews the home's grant IN PLACE (`#grant`'s
|
|
2450
|
+
// renewal branch mutates `existing.token`), so the home can hold a newer token than the one we
|
|
2451
|
+
// installed. Releasing that token would clear the grant still backing our own live delegation
|
|
2452
|
+
// and let the home hand the key to another node while we are inside it. Comparing tokens here
|
|
2453
|
+
// caught only the case where we installed this exact grant.
|
|
2454
|
+
if (held) return;
|
|
2455
|
+
return this.#writeControlSafely({ type: 'lockRelease', key, requester: this.nodeId, token, dependencies: null });
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
#grant(keyId: unknown, key: any, homeMap: LockHomeMap, leaseMs: number, requester: string): DelegationReply {
|
|
2459
|
+
const now = this.#monotonic();
|
|
2460
|
+
this.#prepareFreshnessGeneration(homeMap.generation);
|
|
2461
|
+
// Consulted unconditionally — it is what observes a gap in ownership, and the waiver it applies
|
|
2462
|
+
// to itself is the only part `grantableAfterMono` may switch off.
|
|
2463
|
+
const quarantine = Math.max(this.#grantableAfterMono, this.#ownershipHorizon(now, homeMap.homeIncarnation)) - now;
|
|
2464
|
+
// NOT `contended`: the quarantine runs for a full delegation lease, and `MAX_LOCK_TIMEOUT_MS` is
|
|
2465
|
+
// shorter than that, so retrying it would spend the caller's whole budget and then answer 423 —
|
|
2466
|
+
// "held by someone else" — for a key nobody holds.
|
|
2467
|
+
if (quarantine > 0) return { granted: false, reason: 'quarantine', retryAfterMs: Math.min(quarantine, 250) };
|
|
2468
|
+
const existing = this.#grants.get(keyId);
|
|
2469
|
+
if (existing) {
|
|
2470
|
+
// An expired grant is not a live one. Collecting it here rather than trusting `tick()` is
|
|
2471
|
+
// what keeps a table whose expiry budget is saturated from answering `contended` forever.
|
|
2472
|
+
if (existing.expiresMono <= now) this.#expireGrant(keyId, existing);
|
|
2473
|
+
else if (existing.recalling || existing.recallConfirmed)
|
|
2474
|
+
// A recall is in flight, or the delegate has already confirmed one. Renewing now — even for
|
|
2475
|
+
// the node being recalled — would mint a token its own release no longer matches, and the
|
|
2476
|
+
// contender would never get the key. `recallConfirmed` has to be here as well as in
|
|
2477
|
+
// `#beginRecall`: the delegate can confirm and re-ask before its release reaches the home,
|
|
2478
|
+
// and renewing then would leave a grant nothing will recall again and nothing can release.
|
|
2479
|
+
return { granted: false, reason: 'contended', retryAfterMs: 25 };
|
|
2480
|
+
else if (existing.delegate === requester) {
|
|
2481
|
+
// Renewal for the node that already holds it: extend rather than recall itself.
|
|
2482
|
+
this.#recordGenerationActedOn(homeMap.generation);
|
|
2483
|
+
existing.token = [homeMap.generation, homeMap.homeIncarnation, ++this.#counter];
|
|
2484
|
+
existing.renewed = true;
|
|
2485
|
+
existing.expiresMono = now + DELEGATION_LEASE_MS + this.#skewMs;
|
|
2486
|
+
return {
|
|
2487
|
+
granted: true,
|
|
2488
|
+
token: existing.token,
|
|
2489
|
+
leaseMs: DELEGATION_LEASE_MS,
|
|
2490
|
+
dependencies: existing.dependencies,
|
|
2491
|
+
};
|
|
2492
|
+
} else {
|
|
2493
|
+
// Someone else holds it. Start the recall and make the caller come back — holding the
|
|
2494
|
+
// request open across a drain would tie the home's reply to the previous delegate's
|
|
2495
|
+
// liveness.
|
|
2496
|
+
this.#beginRecall(keyId, existing);
|
|
2497
|
+
return { granted: false, reason: 'contended', retryAfterMs: 25 };
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
if (this.#grants.size >= MAX_DELEGATIONS_PER_TABLE) return { granted: false, reason: 'capacity' };
|
|
2501
|
+
const perRequester = this.#grantsByRequester.get(requester) ?? 0;
|
|
2502
|
+
if (perRequester >= MAX_DELEGATIONS_PER_REQUESTER) return { granted: false, reason: 'capacity' };
|
|
2503
|
+
this.#recordGenerationActedOn(homeMap.generation);
|
|
2504
|
+
const token: FencingToken = [homeMap.generation, homeMap.homeIncarnation, ++this.#counter];
|
|
2505
|
+
const dependencies = this.#freshnessFor(keyId);
|
|
2506
|
+
this.#everDelegated.add(keyId);
|
|
2507
|
+
this.#grants.set(keyId, {
|
|
2508
|
+
key,
|
|
2509
|
+
delegate: requester,
|
|
2510
|
+
token,
|
|
2511
|
+
dependencies,
|
|
2512
|
+
// The home always outwaits the delegate by skew, so it cannot re-grant a key the previous
|
|
2513
|
+
// delegate still believes it holds. The delegation runs for its own fixed duration rather
|
|
2514
|
+
// than the caller's lock lease — a delegation sized to one lock leaves no room for the next
|
|
2515
|
+
// one, and every repeat lock would pay a round trip.
|
|
2516
|
+
expiresMono: now + DELEGATION_LEASE_MS + this.#skewMs,
|
|
2517
|
+
});
|
|
2518
|
+
this.#grantsByRequester.set(requester, perRequester + 1);
|
|
2519
|
+
this.#startTicking();
|
|
2520
|
+
return { granted: true, token, leaseMs: DELEGATION_LEASE_MS, dependencies };
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
#beginRecall(keyId: unknown, grant: HomeGrant): void {
|
|
2524
|
+
if (grant.recalling || grant.recallConfirmed) return;
|
|
2525
|
+
if (grant.recallRetryAfterMono !== undefined && this.#monotonic() < grant.recallRetryAfterMono) return;
|
|
2526
|
+
if (grant.delegate === this.nodeId) {
|
|
2527
|
+
// We are both home and delegate. Recall ourselves through the same path a peer would take,
|
|
2528
|
+
// and settle it the same way: a recall that did not apply must leave the grant recallable,
|
|
2529
|
+
// or `recalling` stays latched here for the rest of the delegation and no contender ever
|
|
2530
|
+
// prompts another one.
|
|
2531
|
+
grant.recalling = this.onDelegationRecall({ key: grant.key, token: grant.token }).then(
|
|
2532
|
+
() => {
|
|
2533
|
+
grant.recalling = undefined;
|
|
2534
|
+
grant.recallConfirmed = true;
|
|
2535
|
+
},
|
|
2536
|
+
(error) => {
|
|
2537
|
+
warnOnce('failed to recall a local record lock delegation', error);
|
|
2538
|
+
grant.recalling = undefined;
|
|
2539
|
+
grant.recallRetryAfterMono = this.#monotonic() + RECALL_RETRY_MS;
|
|
2540
|
+
}
|
|
2541
|
+
);
|
|
2542
|
+
return;
|
|
2543
|
+
}
|
|
2544
|
+
// `.then`, not `Promise.resolve(recallDelegation(...))`: the transport call is evaluated before
|
|
2545
|
+
// `Promise.resolve` and can throw synchronously, which escapes this whole method — so the
|
|
2546
|
+
// handlers below never run, the retry interval is never armed, and the next contender pass
|
|
2547
|
+
// throws again immediately instead of backing off.
|
|
2548
|
+
grant.recalling = Promise.resolve()
|
|
2549
|
+
.then(() =>
|
|
2550
|
+
this.transport.recallDelegation(grant.delegate, this.database, this.table, {
|
|
2551
|
+
key: grant.key,
|
|
2552
|
+
token: grant.token,
|
|
2553
|
+
})
|
|
2554
|
+
)
|
|
2555
|
+
.then(() => {
|
|
2556
|
+
// The delegate confirmed it stopped admitting, so re-sending buys nothing: the grant is
|
|
2557
|
+
// cleared by its release entry, or failing that by its own deadline. Re-arming here is what
|
|
2558
|
+
// let a contender polling at 25 ms fire a recall per pass for the rest of the delegation.
|
|
2559
|
+
grant.recalling = undefined;
|
|
2560
|
+
grant.recallConfirmed = true;
|
|
2561
|
+
})
|
|
2562
|
+
.catch(() => {
|
|
2563
|
+
// An unreachable delegate is not a reason to re-grant early: the grant's own deadline is
|
|
2564
|
+
// what makes the successor safe, and it already includes the skew margin. Retrying IS
|
|
2565
|
+
// worthwhile here — the delegate may come back — but on its own interval, not the
|
|
2566
|
+
// contender's.
|
|
2567
|
+
grant.recalling = undefined;
|
|
2568
|
+
grant.recallRetryAfterMono = this.#monotonic() + RECALL_RETRY_MS;
|
|
2569
|
+
});
|
|
2570
|
+
}
|
|
2571
|
+
|
|
2572
|
+
/**
|
|
2573
|
+
* Give up a delegation: stop admitting, then write the release that lets the home re-grant. One
|
|
2574
|
+
* surrender per delegation, memoized: the home re-sends a recall `RECALL_RETRY_MS` after its own
|
|
2575
|
+
* recall call failed, which is far inside a delegation lease, so two recalls for the same token can
|
|
2576
|
+
* both be waiting on the drain and resume together. Without the memo the second finds the
|
|
2577
|
+
* admissions the first already emptied, has no fence to wait for, and writes the release while the
|
|
2578
|
+
* first is still waiting for a relayed handle to confirm it is fenced.
|
|
2579
|
+
*/
|
|
2580
|
+
#surrender(keyId: unknown, delegation: Delegation): Promise<void> {
|
|
2581
|
+
return (delegation.surrendering ??= this.#surrenderOnce(keyId, delegation));
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
async #surrenderOnce(keyId: unknown, delegation: Delegation): Promise<void> {
|
|
2585
|
+
if (this.#delegations.get(keyId) === delegation) this.#delegations.delete(keyId);
|
|
2586
|
+
// Capability, not just admission: anything this delegation admitted must be unable to commit
|
|
2587
|
+
// before the home is told it may re-grant. A handle admitted on another worker thread revokes
|
|
2588
|
+
// over a message, so wait for its fence to land — bounded by the handle's own lease, past which
|
|
2589
|
+
// it fences itself — before writing the release that lets the home re-grant the key.
|
|
2590
|
+
await this.#revokeAllAndSettle(delegation);
|
|
2591
|
+
const entry: LockReleaseEntry = {
|
|
2592
|
+
type: 'lockRelease',
|
|
2593
|
+
key: delegation.key,
|
|
2594
|
+
requester: this.nodeId,
|
|
2595
|
+
token: delegation.token,
|
|
2596
|
+
dependencies: delegation.dependencies,
|
|
2597
|
+
};
|
|
2598
|
+
return this.#writeControlSafely(entry);
|
|
2599
|
+
}
|
|
2600
|
+
|
|
2601
|
+
/**
|
|
2602
|
+
* Revoke every handle this delegation is answerable for and forget the admissions. Returns, per
|
|
2603
|
+
* admission, the revoker's outcome paired with the admission's own monotonic deadline, so a caller
|
|
2604
|
+
* that must not write the release before the fence is proven (`#surrender`) can bound its wait by
|
|
2605
|
+
* that deadline. Callers tearing the delegation down anyway (`tick`, `close`) ignore the result. A
|
|
2606
|
+
* revoke that throws synchronously is captured as a rejected outcome, never swallowed as fenced.
|
|
2607
|
+
*/
|
|
2608
|
+
#revokeAll(delegation: Delegation): { outcome: void | Promise<void>; expiresMono: number }[] {
|
|
2609
|
+
const admissions = delegation.admissions;
|
|
2610
|
+
delegation.admissions = new Map();
|
|
2611
|
+
delegation.holding = 0;
|
|
2612
|
+
delegation.sweepAtSize = ADMISSION_SWEEP_FLOOR;
|
|
2613
|
+
const outcomes: { outcome: void | Promise<void>; expiresMono: number }[] = [];
|
|
2614
|
+
for (const [admissionId, admission] of admissions) {
|
|
2615
|
+
this.#admissions.delete(admissionId);
|
|
2616
|
+
let outcome: void | Promise<void>;
|
|
2617
|
+
try {
|
|
2618
|
+
outcome = admission.revoke();
|
|
2619
|
+
} catch (error) {
|
|
2620
|
+
// A synchronous throw is NOT a fence: surface it as a rejected outcome so `#revokeAllAndSettle`
|
|
2621
|
+
// waits out the admission's own lease rather than releasing against a handle that may still commit.
|
|
2622
|
+
outcome = Promise.reject(error);
|
|
2623
|
+
}
|
|
2624
|
+
// Every fire-and-forget caller (`tick`, `close`) discards this array, so a rejecting outcome —
|
|
2625
|
+
// a sync throw above, or a relayed revoker whose promise rejects on a dead sibling port — must
|
|
2626
|
+
// carry its own no-op handler here or Node's default policy would exit the worker. The awaited
|
|
2627
|
+
// path (`#revokeAllAndSettle`) still sees the rejection: a settled promise may be awaited again.
|
|
2628
|
+
if (outcome && typeof (outcome as Promise<void>).then === 'function') (outcome as Promise<void>).catch(() => {});
|
|
2629
|
+
outcomes.push({ outcome, expiresMono: admission.expiresMono });
|
|
2630
|
+
}
|
|
2631
|
+
return outcomes;
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2634
|
+
/**
|
|
2635
|
+
* `#revokeAll`, then wait for every asynchronous fence (a handle on another worker acking its
|
|
2636
|
+
* `revokeLease`) before the caller writes the release. Each wait is bounded HERE by the admission's
|
|
2637
|
+
* own remaining lease, independent of what the transport's revoker does: a revoker that rejects,
|
|
2638
|
+
* throws, or never settles cannot make `#surrender` publish the release before the fenced handle's
|
|
2639
|
+
* lease has elapsed — past which the handle fences itself and can no longer commit. The transport's
|
|
2640
|
+
* own lease-bounded ack (harper-pro) is the fast path; this is the guarantee.
|
|
2641
|
+
*/
|
|
2642
|
+
async #revokeAllAndSettle(delegation: Delegation): Promise<void> {
|
|
2643
|
+
const outcomes = this.#revokeAll(delegation);
|
|
2644
|
+
// ONE shared lease timer for the whole settle, not one per admission: a recall of a delegation
|
|
2645
|
+
// holding thousands of relayed admissions would otherwise arm thousands of timers at exactly the
|
|
2646
|
+
// moment it is handing off. The shared deadline is the LATEST lease among them, which bounds every
|
|
2647
|
+
// handle in the set — waiting past a shorter lease only ever errs towards holding the release
|
|
2648
|
+
// longer, never towards writing it early.
|
|
2649
|
+
let latestExpiry = -Infinity;
|
|
2650
|
+
let anyAsync = false;
|
|
2651
|
+
for (const { outcome, expiresMono } of outcomes) {
|
|
2652
|
+
if (!isPromiseLike(outcome)) continue;
|
|
2653
|
+
anyAsync = true;
|
|
2654
|
+
if (expiresMono > latestExpiry) latestExpiry = expiresMono;
|
|
2655
|
+
}
|
|
2656
|
+
// A synchronous revoker has already fenced by the time it returns, so it contributes nothing to
|
|
2657
|
+
// wait on: only the promise-returning ones cost an entry here.
|
|
2658
|
+
if (!anyAsync) return;
|
|
2659
|
+
const leaseTimer = delay(Math.max(0, latestExpiry - this.#monotonic()));
|
|
2660
|
+
const waits: Promise<unknown>[] = [];
|
|
2661
|
+
for (const { outcome } of outcomes) {
|
|
2662
|
+
if (!isPromiseLike(outcome)) continue;
|
|
2663
|
+
// Race the fence ack against the shared lease deadline: whichever comes first, the handle can no
|
|
2664
|
+
// longer commit once we return. A rejected fence is not a confirmed one, so it falls through to
|
|
2665
|
+
// the same deadline rather than resolving early.
|
|
2666
|
+
waits.push(
|
|
2667
|
+
Promise.race([
|
|
2668
|
+
outcome.catch((error) => {
|
|
2669
|
+
warnOnce('a record lock handle did not confirm revocation; waiting out its lease', error);
|
|
2670
|
+
return leaseTimer.promise;
|
|
2671
|
+
}),
|
|
2672
|
+
leaseTimer.promise,
|
|
2673
|
+
])
|
|
2674
|
+
);
|
|
2675
|
+
}
|
|
2676
|
+
try {
|
|
2677
|
+
await Promise.all(waits);
|
|
2678
|
+
} finally {
|
|
2679
|
+
leaseTimer.cancel();
|
|
2680
|
+
}
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
#clearGrant(keyId: unknown, grant: HomeGrant): void {
|
|
2684
|
+
if (this.#grants.get(keyId) !== grant) return;
|
|
2685
|
+
this.#grants.delete(keyId);
|
|
2686
|
+
const count = (this.#grantsByRequester.get(grant.delegate) ?? 1) - 1;
|
|
2687
|
+
if (count > 0) this.#grantsByRequester.set(grant.delegate, count);
|
|
2688
|
+
else this.#grantsByRequester.delete(grant.delegate);
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
#expireGrant(keyId: unknown, grant: HomeGrant): void {
|
|
2692
|
+
if (this.#grants.get(keyId) !== grant) return;
|
|
2693
|
+
this.#dependencySets.delete(keyId);
|
|
2694
|
+
this.#clearGrant(keyId, grant);
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
async #writeControlSafely(entry: LockReleaseEntry): Promise<void> {
|
|
2698
|
+
try {
|
|
2699
|
+
const position = await this.#writeControl(entry);
|
|
2700
|
+
this.#authority().applyEntry(entry, this.nodeId, typeof position === 'number' ? position : undefined);
|
|
2701
|
+
} catch (error) {
|
|
2702
|
+
// A lost release costs the key its remaining lease on the home; it never costs exclusion.
|
|
2703
|
+
warnOnce('failed to write a record lock release entry', error);
|
|
2704
|
+
// A local home can discard its own failed handback safely; its successor will recover.
|
|
2705
|
+
try {
|
|
2706
|
+
const keyId = this.#keyIdOf(entry.key);
|
|
2707
|
+
const grant = this.#grants.get(keyId);
|
|
2708
|
+
if (grant?.delegate === this.nodeId && compareTokens(grant.token, entry.token) === 0)
|
|
2709
|
+
this.#expireGrant(keyId, grant);
|
|
2710
|
+
} catch {}
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
#startTicking() {
|
|
2715
|
+
if (!this.#autoTick) return;
|
|
2716
|
+
tickingCoordinators.add(this);
|
|
2717
|
+
ensureTicking();
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
/**
|
|
2722
|
+
* What ended an exhausted `acquire()`, for the 503 it throws. A `timeout` is not phrased as a home
|
|
2723
|
+
* answer: it is this node's own deadline, and an operator told "the home answered timeout" looks for
|
|
2724
|
+
* a fault on a node that was simply not waited for.
|
|
2725
|
+
*/
|
|
2726
|
+
function describeExhaustedWait(terminal: DelegationReply | undefined, hasCurrentMap: boolean): string {
|
|
2727
|
+
if (terminal)
|
|
2728
|
+
return terminal.reason === 'timeout'
|
|
2729
|
+
? "no reply from the key's home within the wait"
|
|
2730
|
+
: `the key's home answered ${terminal.reason ?? (terminal.granted ? 'grants that arrived too late to use' : 'nothing usable')}`;
|
|
2731
|
+
return hasCurrentMap
|
|
2732
|
+
? 'the record lock home map changed generation before the wait ended'
|
|
2733
|
+
: 'this node has no current record lock home map';
|
|
2734
|
+
}
|
|
2735
|
+
|
|
2736
|
+
/** A sleep whose timer the winner of a race can drop, rather than let it run out its full delay. */
|
|
2737
|
+
function delay(ms: number): { promise: Promise<void>; cancel: () => void } {
|
|
2738
|
+
let timer: ReturnType<typeof setTimeout>;
|
|
2739
|
+
const promise = new Promise<void>((resolve) => {
|
|
2740
|
+
timer = setTimeout(resolve, ms);
|
|
2741
|
+
timer.unref?.();
|
|
2742
|
+
});
|
|
2743
|
+
return { promise, cancel: () => clearTimeout(timer!) };
|
|
2744
|
+
}
|
|
2745
|
+
|
|
2746
|
+
const clusterLockTransports = new Map<string, ClusterLockTransport>();
|
|
2747
|
+
// Databases that have had a transport registered on this worker. A later absence is a transport
|
|
2748
|
+
// that went away — a component reload, a failed reconnect — not proof the node became standalone,
|
|
2749
|
+
// so cluster scope must keep failing closed rather than quietly reverting to a node-local lock.
|
|
2750
|
+
const clusterRequiredDatabases = new Set<string>();
|
|
2751
|
+
type CoordinatorResolver = (database: string, table: string) => LockCoordinator | undefined;
|
|
2752
|
+
let coordinatorResolver: CoordinatorResolver | undefined;
|
|
2753
|
+
// Applying a release is bookkeeping on this node's own grant table and needs no transport, so it
|
|
2754
|
+
// resolves the coordinator that HOLDS the grant rather than the one a registered transport answers
|
|
2755
|
+
// for. Dropping a peer's clean-handoff release during a reconnect denies every other node that key
|
|
2756
|
+
// for the delegation's whole deadline. Requests and recalls keep the transport-gated resolver: both
|
|
2757
|
+
// are answers to a live peer, and failing them closed while the transport is gone is the right shape.
|
|
2758
|
+
let admittingResolver: CoordinatorResolver | undefined;
|
|
2759
|
+
type ControlWriter = (entry: LockControlEntry) => Promise<number | void> | number | void;
|
|
2760
|
+
type ControlWriterResolver = (database: string, table: string) => ControlWriter | undefined;
|
|
2761
|
+
let controlWriterResolver: ControlWriterResolver | undefined;
|
|
2762
|
+
|
|
2763
|
+
/** Installed by Table.ts so a transport can push received entries in without importing Table. */
|
|
2764
|
+
export function setLockCoordinatorResolver(
|
|
2765
|
+
resolve: CoordinatorResolver,
|
|
2766
|
+
resolveAdmitting: CoordinatorResolver = resolve,
|
|
2767
|
+
resolveControlWriter?: ControlWriterResolver
|
|
2768
|
+
) {
|
|
2769
|
+
coordinatorResolver = resolve;
|
|
2770
|
+
admittingResolver = resolveAdmitting;
|
|
2771
|
+
controlWriterResolver = resolveControlWriter;
|
|
2772
|
+
}
|
|
2773
|
+
|
|
2774
|
+
export interface QuiesceOutstanding {
|
|
2775
|
+
table: string;
|
|
2776
|
+
key: unknown;
|
|
2777
|
+
/** Present when this node was the HOME and the delegate did not confirm. */
|
|
2778
|
+
delegate?: string;
|
|
2779
|
+
reason: string;
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
export interface QuiesceResult {
|
|
2783
|
+
/**
|
|
2784
|
+
* Whether this result is a PROOF of quiescence, or merely a report of what was swept.
|
|
2785
|
+
*
|
|
2786
|
+
* A sweep can only visit coordinators that exist on this thread, and they are built lazily — a
|
|
2787
|
+
* table nothing has touched since a restart has none, so an empty `outstanding` would otherwise
|
|
2788
|
+
* read as "nothing is live" when the previous incarnation's delegations are still running
|
|
2789
|
+
* elsewhere. An orchestrator must require `complete && outstanding.length === 0`; anything else
|
|
2790
|
+
* means fall back to the drain interval for this node.
|
|
2791
|
+
*/
|
|
2792
|
+
complete: boolean;
|
|
2793
|
+
/** Delegations this node held and gave up, so it can no longer admit under them. */
|
|
2794
|
+
surrendered: number;
|
|
2795
|
+
/** Grants this node issued whose delegate confirmed it stopped admitting. */
|
|
2796
|
+
recalled: number;
|
|
2797
|
+
/**
|
|
2798
|
+
* What is still live, or unprovable. Empty means this node is provably quiesced for the database —
|
|
2799
|
+
* and ONLY then, which is why a coordinator still inside its restart quarantine contributes an
|
|
2800
|
+
* entry here rather than reporting a clean sweep it cannot back.
|
|
2801
|
+
*/
|
|
2802
|
+
outstanding: QuiesceOutstanding[];
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
/**
|
|
2806
|
+
* Stop this node admitting under the current generation for `database`, and say whether it is
|
|
2807
|
+
* provably done (harper-pro#856).
|
|
2808
|
+
*
|
|
2809
|
+
* A membership change otherwise has to wait out `DELEGATION_LEASE_MS + LOCK_LEASE_SKEW_MS` before the
|
|
2810
|
+
* next generation may activate, because authority already issued under the old one has to expire —
|
|
2811
|
+
* roughly six minutes during which the staged nodes serve no cluster locks at all. That interval is a
|
|
2812
|
+
* TIMER, chosen because you cannot recall what you cannot reach. In a planned transition every
|
|
2813
|
+
* participant is reachable, so the same guarantee can be *established* instead of waited out: this
|
|
2814
|
+
* drains both directions and reports what, if anything, is left.
|
|
2815
|
+
*
|
|
2816
|
+
* - **As a delegate** it surrenders every delegation it holds. This is the load-bearing half: a
|
|
2817
|
+
* delegate is what admits, and `onDelegationRecall` is the existing path whose resolution means
|
|
2818
|
+
* "nothing on this node can admit on that token" — live critical sections are drained, not cut.
|
|
2819
|
+
* - **As a home** it recalls every grant it issued, so delegates elsewhere stop too. Redundant when
|
|
2820
|
+
* every node is quiescing at once, and the reason this still terminates when one is not.
|
|
2821
|
+
*
|
|
2822
|
+
* `outstanding` empty on every node in `homes(g) ∪ homes(g+1)` is the operator's evidence that the
|
|
2823
|
+
* next generation may be activated immediately. Anything left is a node to fall back to the timer for,
|
|
2824
|
+
* or to fence externally — this never claims a drain it did not get, and never throws for one grant.
|
|
2825
|
+
*/
|
|
2826
|
+
export async function quiesceDelegations(database: string, budgetMs: number): Promise<QuiesceResult> {
|
|
2827
|
+
const result: QuiesceResult = { complete: true, surrendered: 0, recalled: 0, outstanding: [] };
|
|
2828
|
+
const coordinators = [...liveCoordinators].filter((coordinator) => coordinator.database === database);
|
|
2829
|
+
// A DURATION, not an absolute deadline: the whole sweep gets this long, measured from here.
|
|
2830
|
+
const deadline = Date.now() + Math.max(0, budgetMs);
|
|
2831
|
+
const remaining = () => Math.max(0, deadline - Date.now());
|
|
2832
|
+
for (const coordinator of coordinators) {
|
|
2833
|
+
await coordinator.quiesce(result, remaining);
|
|
2834
|
+
}
|
|
2835
|
+
// A sweep proves quiescence only if it could have seen everything this NODE issued, not merely what
|
|
2836
|
+
// this coordinator holds. Two ways it could not have:
|
|
2837
|
+
//
|
|
2838
|
+
// - No coordinator exists for the database on this thread, so there is nothing to attest from.
|
|
2839
|
+
// - A coordinator has not owned coordination long enough for authority issued BEFORE it took over
|
|
2840
|
+
// — by a previous owner thread, or a previous incarnation of this process — to have expired.
|
|
2841
|
+
// Those grants live on delegate nodes and no local sweep can see them; the same fact is what
|
|
2842
|
+
// core's own grant gate refuses on, and it is why uptime is not the right measure (a worker that
|
|
2843
|
+
// built empty coordinators early and took ownership late has plenty of uptime and no proof).
|
|
2844
|
+
if (coordinators.length === 0) {
|
|
2845
|
+
result.outstanding.push({
|
|
2846
|
+
table: '*',
|
|
2847
|
+
key: undefined,
|
|
2848
|
+
reason:
|
|
2849
|
+
'no lock coordinator exists for this database on this thread, so there is nothing to prove quiescence from',
|
|
2850
|
+
});
|
|
2851
|
+
}
|
|
2852
|
+
for (const coordinator of coordinators) {
|
|
2853
|
+
const unproven = coordinator.unprovenOwnershipMs();
|
|
2854
|
+
if (unproven > 0)
|
|
2855
|
+
result.outstanding.push({
|
|
2856
|
+
table: coordinator.table,
|
|
2857
|
+
key: undefined,
|
|
2858
|
+
reason: `this thread has not coordinated ${database}.${coordinator.table} long enough to rule out authority issued before it took over; ${Math.ceil(unproven)}ms remain`,
|
|
2859
|
+
});
|
|
2860
|
+
}
|
|
2861
|
+
// A coordinator that closed parked the latest deadline of the grants it had issued to OTHER nodes
|
|
2862
|
+
// here and then cleared its own table (`close`). Those grants are still valid on their delegates and
|
|
2863
|
+
// no live coordinator holds them, so a sweep that ignored this would miss them entirely — the table
|
|
2864
|
+
// may not even have a coordinator any more.
|
|
2865
|
+
//
|
|
2866
|
+
// `performance.now()` because a retired entry outlives the coordinator whose injected clock produced
|
|
2867
|
+
// its deadline: production passes that same clock (the transport's `monotonicNow`), so the domains
|
|
2868
|
+
// agree where it matters, and a test on an artificial clock only ever reads the deadline as further
|
|
2869
|
+
// away — conservative, never a false clean.
|
|
2870
|
+
const now = performance.now();
|
|
2871
|
+
for (const [key, retired] of retiredCoordinators) {
|
|
2872
|
+
const separator = key.indexOf('\u0000');
|
|
2873
|
+
if (separator < 0 || key.slice(0, separator) !== database) continue;
|
|
2874
|
+
if (!(retired.grantableAfterMono > now)) continue;
|
|
2875
|
+
result.outstanding.push({
|
|
2876
|
+
table: key.slice(separator + 1),
|
|
2877
|
+
key: undefined,
|
|
2878
|
+
reason: `a closed coordinator for this table issued grants that remain valid on their delegates for another ${Math.ceil(retired.grantableAfterMono - now)}ms`,
|
|
2879
|
+
});
|
|
2880
|
+
}
|
|
2881
|
+
if (result.outstanding.length > 0) result.complete = false;
|
|
2882
|
+
return result;
|
|
2883
|
+
}
|
|
2884
|
+
|
|
2885
|
+
/**
|
|
2886
|
+
* Commit a `lockBarrier` entry for the table and resolve to its transaction-log position — the §7.2
|
|
2887
|
+
* recovery fence, for a transport answering a peer's recovery probe. The entry is appended after
|
|
2888
|
+
* every transaction this node had committed when the call was made, so a peer that has applied this
|
|
2889
|
+
* origin's log through the returned position has applied all of them.
|
|
2890
|
+
*
|
|
2891
|
+
* The transport supplies the nonce it will match the entry on, since a position alone is not an
|
|
2892
|
+
* identity: a restart after the wall clock moved backwards can reissue a log key, and a drain that
|
|
2893
|
+
* matched the earlier entry at that key would declare this origin drained with its post-restart
|
|
2894
|
+
* commits unapplied.
|
|
2895
|
+
*
|
|
2896
|
+
* Strictly this node's own commit, never the transport's `writeControl`: the fence is a position in
|
|
2897
|
+
* THIS origin's log, and the caller is the transport itself — a relaying hook would answer with a
|
|
2898
|
+
* position that is not local, or re-enter the operation that called here. A write that commits
|
|
2899
|
+
* without a position rejects rather than resolve, since a barrier nobody can wait on is not a fence.
|
|
2900
|
+
*/
|
|
2901
|
+
export async function writeLockBarrier(database: string, table: string, nonce: number): Promise<number> {
|
|
2902
|
+
if (!Number.isSafeInteger(nonce) || nonce < 0)
|
|
2903
|
+
throw new ClientError('A lock barrier nonce must be a non-negative integer');
|
|
2904
|
+
const write = controlWriterResolver?.(database, table);
|
|
2905
|
+
if (!write) throw new ClientError(`Table ${database}.${table} does not exist`, 404);
|
|
2906
|
+
const position = await write({ type: 'lockBarrier', nonce });
|
|
2907
|
+
if (typeof position !== 'number' || !(position >= 0) || !Number.isFinite(position))
|
|
2908
|
+
throw new LockUnavailableError(`the record lock barrier for ${database}.${table} committed without a log position`);
|
|
2909
|
+
return position;
|
|
2910
|
+
}
|
|
2911
|
+
|
|
2912
|
+
/**
|
|
2913
|
+
* Register harper-pro's transport for a database, on THIS thread.
|
|
2914
|
+
*
|
|
2915
|
+
* **It must be registered on every worker that can serve a `lock()`, not only the coordinating one**
|
|
2916
|
+
* — including a dedicated application worker (harper#2524) — and core cannot check that. `clusterRequiredDatabases` is module state, so a worker that never registers never latches —
|
|
2917
|
+
* and a default-scoped `lock()` there takes the Phase 0 node lock alone while a peer runs the cluster
|
|
2918
|
+
* protocol, which is two nodes admitting one key. The `ownsCoordination()` fail-closed path only
|
|
2919
|
+
* reaches a worker that has a transport. Registering everywhere also makes that path the one a
|
|
2920
|
+
* non-owner worker takes, which is what it exists for.
|
|
2921
|
+
*/
|
|
2922
|
+
export function registerClusterLockTransport(database: string, transport: ClusterLockTransport): void {
|
|
2923
|
+
if (
|
|
2924
|
+
typeof transport?.homeMap !== 'function' ||
|
|
2925
|
+
typeof transport?.ownsCoordination !== 'function' ||
|
|
2926
|
+
typeof transport?.requestDelegation !== 'function' ||
|
|
2927
|
+
typeof transport?.recallDelegation !== 'function' ||
|
|
2928
|
+
typeof transport?.establishLockFreshness !== 'function'
|
|
2929
|
+
)
|
|
2930
|
+
throw new ClientError(
|
|
2931
|
+
'A cluster lock transport must provide homeMap(), ownsCoordination(), requestDelegation(), recallDelegation() and establishLockFreshness()'
|
|
2932
|
+
);
|
|
2933
|
+
transport.onControlEntry = (db: string, table: string, entry: LockControlEntry, author: string, position: number) =>
|
|
2934
|
+
deliverLockControlEntry(db, table, entry, author, position);
|
|
2935
|
+
transport.onDelegationRequest = (db: string, table: string, request: DelegationRequest) =>
|
|
2936
|
+
deliverDelegationRequest(db, table, request);
|
|
2937
|
+
transport.onDelegationRecall = (db: string, table: string, recall: DelegationRecall) =>
|
|
2938
|
+
deliverDelegationRecall(db, table, recall);
|
|
2939
|
+
clusterRequiredDatabases.add(database);
|
|
2940
|
+
clusterLockTransports.set(database, transport);
|
|
2941
|
+
}
|
|
2942
|
+
|
|
2943
|
+
export function unregisterClusterLockTransport(database: string, standalone = false): void {
|
|
2944
|
+
clusterLockTransports.delete(database);
|
|
2945
|
+
// Only an explicit statement that the database is no longer clustered clears the requirement.
|
|
2946
|
+
if (standalone) clusterRequiredDatabases.delete(database);
|
|
2947
|
+
}
|
|
2948
|
+
|
|
2949
|
+
/** True once a transport has been registered for this database and no standalone claim has cleared it. */
|
|
2950
|
+
export function isClusterLockRequired(database: string): boolean {
|
|
2951
|
+
return clusterRequiredDatabases.size > 0 && clusterRequiredDatabases.has(database);
|
|
2952
|
+
}
|
|
2953
|
+
|
|
2954
|
+
/**
|
|
2955
|
+
* The registered transport, if any. The `size` check keeps the Phase 0 path free of a map lookup on
|
|
2956
|
+
* every `lock()` in a build where no transport is ever registered.
|
|
2957
|
+
*/
|
|
2958
|
+
export function getClusterLockTransport(database: string): ClusterLockTransport | undefined {
|
|
2959
|
+
if (clusterLockTransports.size === 0) return undefined;
|
|
2960
|
+
return clusterLockTransports.get(database);
|
|
2961
|
+
}
|
|
2962
|
+
|
|
2963
|
+
export function hasClusterLockTransports(): boolean {
|
|
2964
|
+
return clusterLockTransports.size > 0;
|
|
2965
|
+
}
|
|
2966
|
+
|
|
2967
|
+
/**
|
|
2968
|
+
* Resolve a coordinator for an inbound message. The resolver reaches `Table.lockCoordinator`, which
|
|
2969
|
+
* throws when this node's name is unusable — that throw must not escape a receive boundary, or it
|
|
2970
|
+
* reaches the replicated apply loop and drops the enclosing transaction.
|
|
2971
|
+
*/
|
|
2972
|
+
function coordinatorFor(database: string, table: string, resolve = coordinatorResolver): LockCoordinator | undefined {
|
|
2973
|
+
try {
|
|
2974
|
+
return resolve?.(database, table);
|
|
2975
|
+
} catch (error) {
|
|
2976
|
+
warnOnce('could not resolve a record lock coordinator for a received message', error);
|
|
2977
|
+
return undefined;
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
|
|
2981
|
+
export function deliverLockControlEntry(
|
|
2982
|
+
database: string,
|
|
2983
|
+
table: string,
|
|
2984
|
+
entry: LockControlEntry,
|
|
2985
|
+
author: string,
|
|
2986
|
+
position: number
|
|
2987
|
+
): void {
|
|
2988
|
+
coordinatorFor(database, table, admittingResolver)?.applyEntry(entry, author, position);
|
|
2989
|
+
}
|
|
2990
|
+
|
|
2991
|
+
export async function deliverDelegationRequest(
|
|
2992
|
+
database: string,
|
|
2993
|
+
table: string,
|
|
2994
|
+
request: DelegationRequest
|
|
2995
|
+
): Promise<DelegationReply> {
|
|
2996
|
+
const coordinator = coordinatorFor(database, table);
|
|
2997
|
+
if (!coordinator) return { granted: false, reason: 'not-home' };
|
|
2998
|
+
try {
|
|
2999
|
+
return await coordinator.onDelegationRequest(request);
|
|
3000
|
+
} catch (error) {
|
|
3001
|
+
warnOnce('failed to answer a record lock delegation request', error);
|
|
3002
|
+
return { granted: false, reason: 'not-home' };
|
|
3003
|
+
}
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
export async function deliverDelegationRecall(
|
|
3007
|
+
database: string,
|
|
3008
|
+
table: string,
|
|
3009
|
+
recall: DelegationRecall
|
|
3010
|
+
): Promise<void> {
|
|
3011
|
+
// This is core's receiving end of `recallDelegation`, which resolves only "once the delegate has
|
|
3012
|
+
// drained and stopped admitting" — and `#beginRecall` latches `recallConfirmed` on that resolution
|
|
3013
|
+
// and never re-sends. So a recall this thread could not apply has to FAIL rather than resolve:
|
|
3014
|
+
// `coordinatorFor` is the transport-gated resolver and answers undefined through a reconnect, and
|
|
3015
|
+
// the home reading that silence as a drained delegate denies the key to every other node for the
|
|
3016
|
+
// delegation's whole deadline. Failing it leaves the home's `.catch` to retry on RECALL_RETRY_MS.
|
|
3017
|
+
const coordinator = coordinatorFor(database, table);
|
|
3018
|
+
if (!coordinator)
|
|
3019
|
+
throw new Error(`No record lock coordinator on this thread to apply a recall for ${database}.${table}`);
|
|
3020
|
+
await coordinator.onDelegationRecall(recall);
|
|
3021
|
+
}
|
|
3022
|
+
|
|
3023
|
+
// ---- owner-worker relay (harper-pro#852) -------------------------------------------------------
|
|
3024
|
+
|
|
3025
|
+
/**
|
|
3026
|
+
* The owner thread's end of `acquireOnOwner`: mint an admission for a `lock()` served on another
|
|
3027
|
+
* worker thread, and register `revoke` as the way to fence that thread's handle. `revoke` is what the
|
|
3028
|
+
* transport wires to a cross-thread message; a recall or surrender here calls it and waits for the
|
|
3029
|
+
* fence before writing the release. Runs on the coordinating thread, resolved through the transport-
|
|
3030
|
+
* gated resolver so it fails when this thread does not coordinate the database — the same shape as
|
|
3031
|
+
* `deliverDelegationRequest`. Returns the round the calling worker installs as a remote admission.
|
|
3032
|
+
*/
|
|
3033
|
+
export async function acquireForRelay(
|
|
3034
|
+
database: string,
|
|
3035
|
+
table: string,
|
|
3036
|
+
key: any,
|
|
3037
|
+
leaseMs: number,
|
|
3038
|
+
waitMs: number,
|
|
3039
|
+
makeRevoke: (round: LockRound) => () => void | Promise<void>
|
|
3040
|
+
): Promise<LockRound> {
|
|
3041
|
+
const coordinator = coordinatorFor(database, table);
|
|
3042
|
+
if (!coordinator)
|
|
3043
|
+
throw new Error(`No record lock coordinator on this thread to acquire ${database}.${table} for a peer worker`);
|
|
3044
|
+
const round = await coordinator.acquire(key, leaseMs, waitMs);
|
|
3045
|
+
// Re-resolve through the ADMITTING resolver rather than reusing the captured coordinator: a transport
|
|
3046
|
+
// swap during the acquire moves the admission to the successor and empties the predecessor, so
|
|
3047
|
+
// registering on the captured object would eagerly revoke a healthy handle and leave the successor's
|
|
3048
|
+
// admission unfenceable. `Table.ts` does the same via `admittingCoordinator`. The revoker is built
|
|
3049
|
+
// from the round so it names the exact admission when it tells the calling worker to fence its handle.
|
|
3050
|
+
const authority = coordinatorFor(database, table, admittingResolver) ?? coordinator;
|
|
3051
|
+
authority.registerAdmission(round.admissionId, makeRevoke(round));
|
|
3052
|
+
return round;
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
/** The owner thread's end of `releaseOnOwner`: end a relayed admission the owner minted. */
|
|
3056
|
+
export function releaseForRelay(database: string, table: string, key: any, admissionId: number): Promise<void> | void {
|
|
3057
|
+
// The admitting resolver, like a received release: it answers the coordinator that HOLDS the
|
|
3058
|
+
// admission even while a transport is momentarily unregistered, so a release is never dropped.
|
|
3059
|
+
return coordinatorFor(database, table, admittingResolver)?.release(key, admissionId);
|
|
3060
|
+
}
|
|
3061
|
+
|
|
3062
|
+
/**
|
|
3063
|
+
* The calling thread's end of an owner `revoke`: fence the handle for a relayed admission this thread
|
|
3064
|
+
* holds. Resolves once the handle's `revokeLease` has run, so the owner may wait for the fence before
|
|
3065
|
+
* it writes the release. The admitting resolver answers the coordinator that adopted the admission
|
|
3066
|
+
* across a transport swap.
|
|
3067
|
+
*/
|
|
3068
|
+
export function revokeRelayedAdmission(database: string, table: string, admissionId: number): Promise<void> {
|
|
3069
|
+
const coordinator = coordinatorFor(database, table, admittingResolver);
|
|
3070
|
+
// No coordinator to fence against means nothing here can commit under that admission; the fence is
|
|
3071
|
+
// vacuously satisfied and the owner may proceed.
|
|
3072
|
+
return coordinator ? coordinator.revokeRemoteAdmission(admissionId) : Promise.resolve();
|
|
3073
|
+
}
|
|
3074
|
+
|
|
3075
|
+
/**
|
|
3076
|
+
* Fail-closed fence for every relayed handle a table's coordinator holds (harper-pro#852), for when
|
|
3077
|
+
* the owner worker that granted them has exited and its delegations are gone. Harper-pro calls this
|
|
3078
|
+
* per table when it learns the coordinating thread for a database changed.
|
|
3079
|
+
*/
|
|
3080
|
+
export function fenceRelayedAdmissions(database: string, table: string): void {
|
|
3081
|
+
coordinatorFor(database, table, admittingResolver)?.fenceAllRemoteAdmissions();
|
|
3082
|
+
}
|