@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/bin/cliOperations.ts +12 -0
  2. package/bin/restart.ts +66 -6
  3. package/components/Application.ts +1134 -112
  4. package/components/DESIGN.md +421 -0
  5. package/components/OptionsWatcher.ts +368 -102
  6. package/components/Scope.ts +20 -1
  7. package/components/componentLoader.ts +43 -4
  8. package/components/deploymentOperations.ts +4 -1
  9. package/components/deploymentRecorder.ts +9 -2
  10. package/components/mcp/DESIGN.md +109 -0
  11. package/components/mcp/audit.ts +21 -17
  12. package/components/operations.js +284 -52
  13. package/components/operationsValidation.js +49 -2
  14. package/components/packageComponent.ts +25 -1
  15. package/components/requestRestart.ts +11 -0
  16. package/config/DESIGN.md +306 -0
  17. package/config/RootConfigWatcher.ts +191 -37
  18. package/config/configReadRetry.ts +62 -0
  19. package/config/configUtils.ts +78 -26
  20. package/config/parseConfigFile.ts +34 -0
  21. package/config/readConfigFileSync.ts +44 -0
  22. package/config/watcherArming.ts +59 -0
  23. package/config-root.schema.json +4 -0
  24. package/dataLayer/DESIGN.md +179 -0
  25. package/dataLayer/harperBridge/ResourceBridge.ts +28 -2
  26. package/dataLayer/restoreMarker.ts +92 -25
  27. package/dist/bin/cliOperations.js +13 -0
  28. package/dist/bin/cliOperations.js.map +1 -1
  29. package/dist/bin/restart.js +42 -6
  30. package/dist/bin/restart.js.map +1 -1
  31. package/dist/components/Application.d.ts +104 -9
  32. package/dist/components/Application.js +954 -102
  33. package/dist/components/Application.js.map +1 -1
  34. package/dist/components/OptionsWatcher.d.ts +4 -1
  35. package/dist/components/OptionsWatcher.js +378 -104
  36. package/dist/components/OptionsWatcher.js.map +1 -1
  37. package/dist/components/Scope.js +15 -1
  38. package/dist/components/Scope.js.map +1 -1
  39. package/dist/components/componentLoader.js +35 -3
  40. package/dist/components/componentLoader.js.map +1 -1
  41. package/dist/components/deploymentOperations.js +4 -1
  42. package/dist/components/deploymentOperations.js.map +1 -1
  43. package/dist/components/deploymentRecorder.d.ts +4 -2
  44. package/dist/components/deploymentRecorder.js +1 -0
  45. package/dist/components/deploymentRecorder.js.map +1 -1
  46. package/dist/components/mcp/audit.d.ts +2 -1
  47. package/dist/components/mcp/audit.js +21 -17
  48. package/dist/components/mcp/audit.js.map +1 -1
  49. package/dist/components/operations.d.ts +28 -0
  50. package/dist/components/operations.js +263 -54
  51. package/dist/components/operations.js.map +1 -1
  52. package/dist/components/operationsValidation.js +48 -2
  53. package/dist/components/operationsValidation.js.map +1 -1
  54. package/dist/components/packageComponent.js +24 -0
  55. package/dist/components/packageComponent.js.map +1 -1
  56. package/dist/components/requestRestart.d.ts +1 -0
  57. package/dist/components/requestRestart.js +7 -0
  58. package/dist/components/requestRestart.js.map +1 -1
  59. package/dist/config/RootConfigWatcher.d.ts +2 -0
  60. package/dist/config/RootConfigWatcher.js +189 -35
  61. package/dist/config/RootConfigWatcher.js.map +1 -1
  62. package/dist/config/configReadRetry.d.ts +8 -0
  63. package/dist/config/configReadRetry.js +62 -0
  64. package/dist/config/configReadRetry.js.map +1 -0
  65. package/dist/config/configUtils.d.ts +10 -9
  66. package/dist/config/configUtils.js +63 -27
  67. package/dist/config/configUtils.js.map +1 -1
  68. package/dist/config/parseConfigFile.d.ts +4 -0
  69. package/dist/config/parseConfigFile.js +35 -0
  70. package/dist/config/parseConfigFile.js.map +1 -0
  71. package/dist/config/readConfigFileSync.d.ts +1 -0
  72. package/dist/config/readConfigFileSync.js +47 -0
  73. package/dist/config/readConfigFileSync.js.map +1 -0
  74. package/dist/config/watcherArming.d.ts +15 -0
  75. package/dist/config/watcherArming.js +59 -0
  76. package/dist/config/watcherArming.js.map +1 -0
  77. package/dist/dataLayer/harperBridge/ResourceBridge.js +21 -2
  78. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  79. package/dist/dataLayer/restoreMarker.d.ts +21 -8
  80. package/dist/dataLayer/restoreMarker.js +94 -27
  81. package/dist/dataLayer/restoreMarker.js.map +1 -1
  82. package/dist/index.d.ts +2 -0
  83. package/dist/index.js +9 -1
  84. package/dist/index.js.map +1 -1
  85. package/dist/json/systemSchema.json +3 -0
  86. package/dist/resources/DatabaseTransaction.d.ts +25 -0
  87. package/dist/resources/DatabaseTransaction.js +224 -18
  88. package/dist/resources/DatabaseTransaction.js.map +1 -1
  89. package/dist/resources/LMDBTransaction.d.ts +2 -1
  90. package/dist/resources/LMDBTransaction.js +22 -3
  91. package/dist/resources/LMDBTransaction.js.map +1 -1
  92. package/dist/resources/PrimaryRocksDatabase.js +22 -6
  93. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  94. package/dist/resources/RecordEncoder.d.ts +1 -1
  95. package/dist/resources/RecordEncoder.js +46 -10
  96. package/dist/resources/RecordEncoder.js.map +1 -1
  97. package/dist/resources/Resource.js +97 -13
  98. package/dist/resources/Resource.js.map +1 -1
  99. package/dist/resources/ResourceInterface.d.ts +8 -0
  100. package/dist/resources/RocksIndexStore.js +2 -1
  101. package/dist/resources/RocksIndexStore.js.map +1 -1
  102. package/dist/resources/RocksTransactionLogStore.d.ts +10 -0
  103. package/dist/resources/RocksTransactionLogStore.js +104 -33
  104. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  105. package/dist/resources/Table.d.ts +58 -7
  106. package/dist/resources/Table.js +1143 -348
  107. package/dist/resources/Table.js.map +1 -1
  108. package/dist/resources/analytics/write.d.ts +3 -0
  109. package/dist/resources/analytics/write.js +49 -16
  110. package/dist/resources/analytics/write.js.map +1 -1
  111. package/dist/resources/auditStore.d.ts +170 -0
  112. package/dist/resources/auditStore.js +457 -11
  113. package/dist/resources/auditStore.js.map +1 -1
  114. package/dist/resources/crdt.d.ts +10 -0
  115. package/dist/resources/crdt.js +22 -0
  116. package/dist/resources/crdt.js.map +1 -1
  117. package/dist/resources/dataLoader.js +3 -4
  118. package/dist/resources/dataLoader.js.map +1 -1
  119. package/dist/resources/databases.d.ts +16 -13
  120. package/dist/resources/databases.js +625 -177
  121. package/dist/resources/databases.js.map +1 -1
  122. package/dist/resources/derivedIndexRegistry.d.ts +5 -0
  123. package/dist/resources/derivedIndexRegistry.js +68 -0
  124. package/dist/resources/derivedIndexRegistry.js.map +1 -0
  125. package/dist/resources/derivedIndexRuntime.d.ts +215 -0
  126. package/dist/resources/derivedIndexRuntime.js +2027 -0
  127. package/dist/resources/derivedIndexRuntime.js.map +1 -0
  128. package/dist/resources/graphql.js +3 -2
  129. package/dist/resources/graphql.js.map +1 -1
  130. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +102 -9
  131. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +867 -39
  132. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  133. package/dist/resources/indexes/fullTextDerivedIndex.d.ts +81 -0
  134. package/dist/resources/indexes/fullTextDerivedIndex.js +1004 -0
  135. package/dist/resources/indexes/fullTextDerivedIndex.js.map +1 -0
  136. package/dist/resources/indexes/fullTextNativeBinding.d.ts +78 -0
  137. package/dist/resources/indexes/fullTextNativeBinding.js +85 -0
  138. package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -0
  139. package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
  140. package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
  141. package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
  142. package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
  143. package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
  144. package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
  145. package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.d.ts +24 -0
  146. package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js +149 -0
  147. package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js.map +1 -0
  148. package/dist/resources/nodeIdMapping.d.ts +5 -0
  149. package/dist/resources/nodeIdMapping.js +49 -0
  150. package/dist/resources/nodeIdMapping.js.map +1 -1
  151. package/dist/resources/recordLock.d.ts +47 -4
  152. package/dist/resources/recordLock.js +138 -7
  153. package/dist/resources/recordLock.js.map +1 -1
  154. package/dist/resources/recordLockCoordinator.d.ts +557 -0
  155. package/dist/resources/recordLockCoordinator.js +2597 -0
  156. package/dist/resources/recordLockCoordinator.js.map +1 -0
  157. package/dist/resources/replayLogs.js +5 -0
  158. package/dist/resources/replayLogs.js.map +1 -1
  159. package/dist/resources/replicatedApplyFailure.d.ts +16 -0
  160. package/dist/resources/replicatedApplyFailure.js +63 -0
  161. package/dist/resources/replicatedApplyFailure.js.map +1 -0
  162. package/dist/resources/scheduler/scheduler.js +3 -3
  163. package/dist/resources/scheduler/scheduler.js.map +1 -1
  164. package/dist/resources/search.d.ts +10 -4
  165. package/dist/resources/search.js +160 -40
  166. package/dist/resources/search.js.map +1 -1
  167. package/dist/resources/tracked.d.ts +5 -1
  168. package/dist/resources/tracked.js +74 -23
  169. package/dist/resources/tracked.js.map +1 -1
  170. package/dist/resources/transactionBroadcast.js +4 -6
  171. package/dist/resources/transactionBroadcast.js.map +1 -1
  172. package/dist/security/auth.js +59 -23
  173. package/dist/security/auth.js.map +1 -1
  174. package/dist/security/deferredAuthentication.d.ts +11 -0
  175. package/dist/security/deferredAuthentication.js +25 -3
  176. package/dist/security/deferredAuthentication.js.map +1 -1
  177. package/dist/security/jsLoader.js +9 -4
  178. package/dist/security/jsLoader.js.map +1 -1
  179. package/dist/server/REST.js +39 -5
  180. package/dist/server/REST.js.map +1 -1
  181. package/dist/server/http.d.ts +5 -1
  182. package/dist/server/http.js +34 -2
  183. package/dist/server/http.js.map +1 -1
  184. package/dist/server/mqtt.js +5 -1
  185. package/dist/server/mqtt.js.map +1 -1
  186. package/dist/server/serverHelpers/Headers.d.ts +2 -0
  187. package/dist/server/serverHelpers/Headers.js +6 -0
  188. package/dist/server/serverHelpers/Headers.js.map +1 -1
  189. package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
  190. package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
  191. package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
  192. package/dist/server/serverHelpers/Request.d.ts +5 -10
  193. package/dist/server/serverHelpers/Request.js +38 -136
  194. package/dist/server/serverHelpers/Request.js.map +1 -1
  195. package/dist/server/serverHelpers/contentTypes.d.ts +2 -0
  196. package/dist/server/serverHelpers/contentTypes.js +189 -15
  197. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  198. package/dist/server/serverHelpers/serverUtilities.d.ts +3 -3
  199. package/dist/server/serverHelpers/serverUtilities.js +110 -20
  200. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  201. package/dist/server/serverHelpers/uwsServer.js +4 -1
  202. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  203. package/dist/server/serverHelpers/webSocketCloseReason.d.ts +2 -0
  204. package/dist/server/serverHelpers/webSocketCloseReason.js +29 -0
  205. package/dist/server/serverHelpers/webSocketCloseReason.js.map +1 -0
  206. package/dist/server/storageReclamation.js +1 -1
  207. package/dist/server/storageReclamation.js.map +1 -1
  208. package/dist/server/threads/isolatedApplications.d.ts +47 -0
  209. package/dist/server/threads/isolatedApplications.js +171 -0
  210. package/dist/server/threads/isolatedApplications.js.map +1 -0
  211. package/dist/server/threads/logRotationTransport.d.ts +1 -0
  212. package/dist/server/threads/logRotationTransport.js +33 -0
  213. package/dist/server/threads/logRotationTransport.js.map +1 -0
  214. package/dist/server/threads/manageThreads.d.ts +64 -6
  215. package/dist/server/threads/manageThreads.js +261 -12
  216. package/dist/server/threads/manageThreads.js.map +1 -1
  217. package/dist/server/threads/socketRouter.d.ts +1 -0
  218. package/dist/server/threads/socketRouter.js +196 -13
  219. package/dist/server/threads/socketRouter.js.map +1 -1
  220. package/dist/server/threads/threadServer.js +30 -7
  221. package/dist/server/threads/threadServer.js.map +1 -1
  222. package/dist/utility/errors/hdbError.d.ts +24 -0
  223. package/dist/utility/errors/hdbError.js +58 -1
  224. package/dist/utility/errors/hdbError.js.map +1 -1
  225. package/dist/utility/hdbTerms.d.ts +2 -0
  226. package/dist/utility/hdbTerms.js +2 -0
  227. package/dist/utility/hdbTerms.js.map +1 -1
  228. package/dist/utility/logging/harper_logger.js +218 -38
  229. package/dist/utility/logging/harper_logger.js.map +1 -1
  230. package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
  231. package/dist/utility/logging/logGenerationCoordinator.js +184 -0
  232. package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
  233. package/dist/utility/logging/logRotation.d.ts +46 -0
  234. package/dist/utility/logging/logRotation.js +365 -0
  235. package/dist/utility/logging/logRotation.js.map +1 -0
  236. package/dist/utility/logging/logRotator.d.ts +1 -1
  237. package/dist/utility/logging/logRotator.js +172 -92
  238. package/dist/utility/logging/logRotator.js.map +1 -1
  239. package/dist/utility/npmUtilities.js +6 -4
  240. package/dist/utility/npmUtilities.js.map +1 -1
  241. package/dist/utility/watcherFallback.d.ts +0 -45
  242. package/dist/utility/watcherFallback.js +1 -125
  243. package/dist/utility/watcherFallback.js.map +1 -1
  244. package/dist/validation/configValidator.js +6 -3
  245. package/dist/validation/configValidator.js.map +1 -1
  246. package/index.ts +9 -0
  247. package/json/systemSchema.json +3 -0
  248. package/npm-shrinkwrap.json +177 -87
  249. package/package.json +13 -5
  250. package/resources/DESIGN.md +689 -19
  251. package/resources/DatabaseTransaction.ts +230 -17
  252. package/resources/LMDBTransaction.ts +21 -3
  253. package/resources/PrimaryRocksDatabase.ts +20 -7
  254. package/resources/RecordEncoder.ts +52 -10
  255. package/resources/Resource.ts +97 -13
  256. package/resources/ResourceInterface.ts +8 -0
  257. package/resources/RocksIndexStore.ts +2 -1
  258. package/resources/RocksTransactionLogStore.ts +111 -31
  259. package/resources/Table.ts +1273 -396
  260. package/resources/analytics/DESIGN.md +38 -0
  261. package/resources/analytics/write.ts +50 -17
  262. package/resources/auditStore.ts +460 -11
  263. package/resources/crdt.ts +22 -0
  264. package/resources/dataLoader.ts +3 -4
  265. package/resources/databases.ts +611 -147
  266. package/resources/derivedIndexRegistry.ts +56 -0
  267. package/resources/derivedIndexRuntime.ts +2292 -0
  268. package/resources/graphql.ts +3 -2
  269. package/resources/indexes/DESIGN.md +833 -0
  270. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +924 -46
  271. package/resources/indexes/fullTextDerivedIndex.ts +1165 -0
  272. package/resources/indexes/fullTextNativeBinding.ts +146 -0
  273. package/resources/indexes/hnswDerivedIndex.ts +531 -0
  274. package/resources/indexes/hnswPlaneBinding.ts +174 -0
  275. package/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts +181 -0
  276. package/resources/nodeIdMapping.ts +50 -0
  277. package/resources/record-locks.md +1407 -0
  278. package/resources/recordLock.ts +173 -7
  279. package/resources/recordLockCoordinator.ts +3082 -0
  280. package/resources/replayLogs.ts +5 -0
  281. package/resources/replicatedApplyFailure.ts +77 -0
  282. package/resources/scheduler/DESIGN.md +40 -0
  283. package/resources/scheduler/scheduler.ts +4 -4
  284. package/resources/search.ts +169 -49
  285. package/resources/tracked.ts +73 -22
  286. package/resources/transactionBroadcast.ts +4 -4
  287. package/security/DESIGN.md +175 -0
  288. package/security/auth.ts +53 -24
  289. package/security/deferredAuthentication.ts +24 -2
  290. package/security/jsLoader.ts +9 -4
  291. package/server/DESIGN.md +275 -0
  292. package/server/REST.ts +42 -6
  293. package/server/http.ts +34 -2
  294. package/server/mqtt.ts +6 -4
  295. package/server/serverHelpers/Headers.ts +5 -1
  296. package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
  297. package/server/serverHelpers/Request.ts +33 -131
  298. package/server/serverHelpers/contentTypes.ts +188 -15
  299. package/server/serverHelpers/serverUtilities.ts +157 -27
  300. package/server/serverHelpers/uwsServer.ts +4 -1
  301. package/server/serverHelpers/webSocketCloseReason.ts +25 -0
  302. package/server/storageReclamation.ts +2 -2
  303. package/server/threads/isolatedApplications.ts +157 -0
  304. package/server/threads/logRotationTransport.ts +40 -0
  305. package/server/threads/manageThreads.js +254 -12
  306. package/server/threads/socketRouter.ts +217 -11
  307. package/server/threads/threadServer.js +30 -7
  308. package/studio/web/assets/{Chat-BnCBegQz.js → Chat-DADFFGe_.js} +1 -1
  309. package/studio/web/assets/{FloatingChat-CoDW1ySS.js → FloatingChat-D_mI-rZ7.js} +3 -3
  310. package/studio/web/assets/{apiToken-Bwk5BLXW.js → apiToken-c2NiSDHa.js} +1 -1
  311. package/studio/web/assets/{applications-DHxGi7JH.js → applications-DktUqh7G.js} +1 -1
  312. package/studio/web/assets/{cssMode-s0cWI-_M.js → cssMode-Cs_75Xhw.js} +1 -1
  313. package/studio/web/assets/{editor-DNcRHK54.js → editor-19b-Y1IN.js} +1 -1
  314. package/studio/web/assets/{html-Bdssedlg.js → html-DiYEQMpB.js} +1 -1
  315. package/studio/web/assets/{htmlMode-CoDlJ3fw.js → htmlMode-CmR0y7P_.js} +1 -1
  316. package/studio/web/assets/{index-D6sxmFLR.js → index-Dm0rfkJ7.js} +5 -5
  317. package/studio/web/assets/{index.lazy-tmU5BS8s.js → index.lazy-7vqt2CC3.js} +1 -1
  318. package/studio/web/assets/{javascript-B8meVSTH.js → javascript-BWtCFuOt.js} +1 -1
  319. package/studio/web/assets/{jsonMode-DpIPd35T.js → jsonMode-Buzzbv9y.js} +1 -1
  320. package/studio/web/assets/{languageServices-C_5FMJzQ.js → languageServices-SqsFWfTM.js} +1 -1
  321. package/studio/web/assets/{lspLanguageFeatures-BIzNBzPK.js → lspLanguageFeatures-EMV5cmjo.js} +1 -1
  322. package/studio/web/assets/{notifications-CvZivSbh.js → notifications-CAB-LZWT.js} +1 -1
  323. package/studio/web/assets/{notifications-CQf18QKb.js → notifications-DRzmSRxM.js} +1 -1
  324. package/studio/web/assets/{profile-DdOwtntb.js → profile-BNKAl79n.js} +1 -1
  325. package/studio/web/assets/{regions-n69fwagr.js → regions-CUow_Zw2.js} +1 -1
  326. package/studio/web/assets/{register-PfWTCXWB.js → register-Dkt3WUMp.js} +2 -2
  327. package/studio/web/assets/{setComponentFile-Bg6O7X0S.js → setComponentFile-BZRfMD0N.js} +1 -1
  328. package/studio/web/assets/{setup-CUx_aUDl.js → setup-D_yiEPO2.js} +2 -2
  329. package/studio/web/assets/{status-D7BVKqX9.js → status-DhHh1Ge-.js} +1 -1
  330. package/studio/web/assets/{toggleHighContrast-DBSyXzMr.js → toggleHighContrast-D7L1PDtV.js} +1 -1
  331. package/studio/web/assets/{tsMode-BByKCjBS.js → tsMode-CCwLk1YS.js} +1 -1
  332. package/studio/web/assets/{typescript-DDLnLpw9.js → typescript-BP1j1mjn.js} +1 -1
  333. package/studio/web/assets/{useEntityRestURL-31CHGaHk.js → useEntityRestURL-D7bnYxLw.js} +1 -1
  334. package/studio/web/assets/{workers-pR3jRY9D.js → workers-tOuCNT17.js} +1 -1
  335. package/studio/web/assets/{xml-2iRnMhQO.js → xml-BSG_3mQT.js} +1 -1
  336. package/studio/web/assets/{yaml-Bf92gJpd.js → yaml-DxiLprBB.js} +1 -1
  337. package/studio/web/index.html +1 -1
  338. package/utility/DESIGN.md +55 -0
  339. package/utility/errors/hdbError.ts +54 -0
  340. package/utility/hdbTerms.ts +2 -0
  341. package/utility/logging/harper_logger.ts +210 -30
  342. package/utility/logging/logGenerationCoordinator.ts +196 -0
  343. package/utility/logging/logRotation.ts +367 -0
  344. package/utility/logging/logRotator.ts +196 -91
  345. package/utility/npmUtilities.ts +6 -4
  346. package/utility/watcherFallback.ts +0 -122
  347. package/validation/configValidator.ts +6 -3
@@ -0,0 +1,3082 @@
1
+ import { performance } from 'node:perf_hooks';
2
+ import { Packr } from 'msgpackr';
3
+ import harperLogger from '../utility/logging/harper_logger.ts';
4
+ import { ClientError, LockUnavailableError } from '../utility/errors/hdbError.ts';
5
+ import { MAX_LOCK_LEASE_MS, MIN_LOCK_LEASE_MS } from './recordLock.ts';
6
+
7
+ /**
8
+ * Cluster-wide record locks (harper#483, Phase 1): amortized per-record ownership.
9
+ * Design note: `docs/record-lock-ownership.md`.
10
+ *
11
+ * `Table.lock()` acquires the node's rocksdb-js key lock first, which bounds this process to one
12
+ * outstanding acquisition per key; only then does it run the cluster step here. That step has three
13
+ * levels at three very different rates:
14
+ *
15
+ * - **The home map** — `(generation, homes[])`, published by an operator through harper-pro and
16
+ * handed to core through `transport.homeMap()`. It is immutable for the life of its generation:
17
+ * core never computes it, never advances it, and never proceeds without it.
18
+ * - **The home node** — within a generation, a key's arbiter is a rendezvous hash over `homes[]`
19
+ * (§4.4). One arbiter per key is trivially exclusive, so there is no grant state machine at all:
20
+ * no deferral queues, no `(tsR, nodeId)` tiebreak, no synthesized grants, no split votes, no
21
+ * revocation protocol between peers.
22
+ * - **The delegation** — the exclusive right to admit critical sections on one key for a bounded
23
+ * time. While one is live, `lock()`/`unlock()` are pure Phase 0: the local key lock and **zero
24
+ * cluster messages**. Releasing the application lock does not release the delegation, so a node
25
+ * writing the same record repeatedly pays one round and then nothing.
26
+ *
27
+ * Two properties carry the safety argument, and both are enforced rather than assumed:
28
+ *
29
+ * - **One delegate per key per home.** A home never has two live delegations for a key, and a
30
+ * successor delegation is issued only after the predecessor's has been recalled-and-drained or has
31
+ * provably expired on the home's own clock plus skew. Every expiry decision on both sides is made
32
+ * on that side's monotonic clock; no remote timestamp is ever compared against a local one.
33
+ * - **A delegation bounds every handle it admitted.** An admission may not outlive its delegation, so
34
+ * recall revokes capability rather than merely closing the door (§6): the commit-time lease fence
35
+ * in `DatabaseTransaction` rejects a staged write whose handle has expired, and a recall expires
36
+ * those handles before the release is acknowledged.
37
+ *
38
+ * `nodeId` here is the globally stable node NAME. It is deliberately not the audit entry's `nodeId`:
39
+ * `nodeIdMapping.ts` hands out per-node short ids (0 is always local), so the same node has different
40
+ * ids on different nodes and any ordering built on them would order the same pair differently on two
41
+ * nodes.
42
+ *
43
+ * Successor freshness follows §7: a clean release carries inherited origin-log dependencies, the
44
+ * home advances the releasing origin to that entry's own position, and the next delegate cannot
45
+ * admit until its transport has made the set visible. Missing lineage takes the weaker recovery
46
+ * barrier and fails closed if that barrier cannot be established.
47
+ */
48
+
49
+ /**
50
+ * How long a home waits before re-sending a recall that FAILED. A recall the delegate confirmed is
51
+ * never re-sent: it has stopped admitting, and the grant is then cleared by its release or by its own
52
+ * deadline. Without both rules a contender polling at 25 ms re-armed the recall on every pass and
53
+ * turned one handoff into an RPC storm lasting the rest of the delegation.
54
+ */
55
+ export const RECALL_RETRY_MS = 1_000;
56
+
57
+ /** Margin a home adds to a delegation it issued, so the delegate always stops admitting first. */
58
+ export const LOCK_LEASE_SKEW_MS = 5_000;
59
+ /**
60
+ * How long a delegation runs, independent of any one caller's lock lease. It MUST be longer than the
61
+ * longest lease it will admit, or the amortization does not exist: a delegation sized to the caller's
62
+ * lease has no room left for the next lock, so every repeat `lock()` renews and pays a round trip —
63
+ * exactly the cost this design is built to remove.
64
+ */
65
+ export const DELEGATION_LEASE_MS = MAX_LOCK_LEASE_MS + 60_000;
66
+ /**
67
+ * Below this an admission map is too small for its dead entries to matter, so the expiry sweep in
68
+ * `#pruneAdmissions` does not run at all; above it the map may reach twice the live set first.
69
+ */
70
+ const ADMISSION_SWEEP_FLOOR = 64;
71
+ const TICK_INTERVAL_MS = 100;
72
+ /**
73
+ * How often `tick()` asks the transport whether this thread still coordinates. Far coarser than the
74
+ * tick because the answer only has to be sampled faster than a gap can matter, and it cannot matter
75
+ * below a full delegation lease: whatever coordinated during the gap is inside its own quarantine
76
+ * until then. At the tick rate an idle coordinating table would call the transport 10 times a second
77
+ * for the life of the process.
78
+ */
79
+ const OWNERSHIP_POLL_MS = 1_000;
80
+ const WARN_INTERVAL_MS = 60_000;
81
+ /**
82
+ * Bounds the grants ONE TABLE's coordinator can accumulate — a home may never forget a delegation
83
+ * before its expiry, so the only bound available is a refusal to issue more. There is one coordinator
84
+ * per table, so the process-wide exposure is this times the number of tables under lock pressure; a
85
+ * true process-level bound is harper#2581 and is sized with the enablement measurements.
86
+ */
87
+ const MAX_DELEGATIONS_PER_TABLE = 10_000;
88
+ /** Bounds what a single peer can make one table's home retain, so one node cannot exhaust it. */
89
+ const MAX_DELEGATIONS_PER_REQUESTER = 2_000;
90
+ /** Bounds expiry work per tick, so a burst of expiries cannot stall the event loop. */
91
+ const MAX_EXPIRIES_PER_TICK = 256;
92
+ /** How many entries a relay sweep may LOOK at per tick, so a thread holding many live relayed
93
+ * admissions does not walk all of them every 100 ms to find nothing. Well above any realistic count of
94
+ * concurrent off-owner locks on one thread, so an ordinary sweep still completes a full pass. */
95
+ const MAX_EXAMINED_PER_TICK = 4_096;
96
+ /** How long past its lease a relayed admission entry is kept before pruning (harper-pro#852), so the
97
+ * handle's own lease timer fires first and forwards its release to the owner rather than racing the
98
+ * sweep that would drop the entry it needs. A few ticks is plenty. */
99
+ const REMOTE_PRUNE_GRACE_MS = 500;
100
+ /**
101
+ * What an owner-worker acquire (harper-pro#852) reserves out of the caller's `waitMs` for the round
102
+ * trip, so the transport is asked for a wait it can answer WITHIN the caller's budget. `lock()` holds
103
+ * the native key for the whole wait, so the hop must come out of that budget, never on top of it:
104
+ * overshooting blocks every other worker on the key for the overshoot. Halved for a caller whose wait
105
+ * is shorter than the allowance, so a short wait still leaves the owner something to work with.
106
+ *
107
+ * Small on purpose. This covers only the core-to-transport boundary: a transport that relays bounds
108
+ * itself inside the wait it is handed and reserves its own margin for the hops it makes, so reserving
109
+ * a hop-sized allowance here as well would subtract the same round trip from the caller twice.
110
+ */
111
+ const REMOTE_ACQUIRE_HOP_MS = 500;
112
+ /** Core's last-resort net on an owner-worker acquire, on top of the caller's `waitMs`. The transport
113
+ * already bounds itself inside that budget, so this fires only when it never returns at all — small,
114
+ * because it is a wedged-transport net and not a second full wait. */
115
+ const REMOTE_ACQUIRE_BACKSTOP_MS = 1_000;
116
+ const MAX_NODE_NAME_LENGTH = 255;
117
+ const MAX_LOCK_DEPENDENCIES = 1_024;
118
+ const MAX_DEPENDENCY_SETS_PER_TABLE = 20_000;
119
+ /**
120
+ * A 256 KiB filter keeps the false-positive rate near 1% through 200,000 distinct delegated keys
121
+ * per lock-active table and generation. Saturation remains safe — it selects recovery — but making
122
+ * that the normal path would turn every first lock on a new key into a cluster-wide barrier.
123
+ */
124
+ const DELEGATED_KEY_FILTER_WORDS = 65_536;
125
+ /** A node whose identity resolved to one of these is not distinctive enough to be a ring member. */
126
+ const NON_DISTINCTIVE_NODE_NAMES = new Set(['127.0.0.1', '::1', 'localhost', '0.0.0.0']);
127
+
128
+ /**
129
+ * The two control entries. `lockRequest`/`lockGrant` belonged to the Ricart–Agrawala rule the
130
+ * design note replaces; they never shipped enabled, so their nibbles were retired rather than
131
+ * migrated (`auditStore.ts`). Delegation request/grant/recall are unicast over the transport, not
132
+ * entries — the release stays on the replicated log because it is what orders a handoff behind the
133
+ * delegate's own data writes, and the barrier is on it because being replicated is its whole
134
+ * purpose (§7.2): a member commits one on request, and its position is the point a peer must have
135
+ * applied that origin through before a recovery-mode successor may admit.
136
+ */
137
+ export type LockControlType = 'lockRelease' | 'lockBarrier';
138
+
139
+ /**
140
+ * The operator-agreed map a key's home is derived from. Supplied by harper-pro; core never computes
141
+ * it and never advances it. Immutable for the life of a generation — nothing a node observes changes
142
+ * it, which is why no agreement protocol runs here (§4).
143
+ */
144
+ export interface LockHomeMap {
145
+ /** Monotonic per database. Part of the fencing token, so it must never go backwards. */
146
+ generation: number;
147
+ /**
148
+ * Every node that participates in cluster record locks for this database — not only the ones an
149
+ * operator thinks of as arbiters. Order is irrelevant; the ring hashes each name independently.
150
+ *
151
+ * It is one set and not two because a home refuses a delegation to any node this list does not
152
+ * name (that is what keeps a decommissioned node from taking one), so a node absent from it can
153
+ * neither home a key nor lock one. Rendezvous hashing then makes every listed node the arbiter for
154
+ * its share of the ring, which is the property that costs a second list nothing.
155
+ */
156
+ homes: string[];
157
+ /**
158
+ * This node's durably persisted, monotonic incarnation counter as a home (§5.1). A random value
159
+ * makes a stale reply identifiable but not ORDERABLE: a home that restarts and re-issues counter 1
160
+ * after having issued counter 50 would let a delayed counter-50 write defeat its successor.
161
+ *
162
+ * **Per coordination incarnation, not per process.** Coordinator state — including the delegation
163
+ * counter — is per-thread, so a replacement coordinating worker starts counting from zero. If the
164
+ * incarnation did not advance with it, the new worker would re-mint tokens its predecessor already
165
+ * issued, and §4.3's incarnation-bound quiescence acknowledgements would survive a restart that
166
+ * discarded everything they attested to.
167
+ */
168
+ homeIncarnation: number;
169
+ }
170
+
171
+ /**
172
+ * A delegation's fencing token, ordered lexicographically as
173
+ * `(generation, homeIncarnation, counter)`. Comparable across homes only within a generation, which
174
+ * is all that is needed: a key has exactly one home per generation.
175
+ */
176
+ export type FencingToken = readonly [generation: number, homeIncarnation: number, counter: number];
177
+ export type LockDependency = readonly [origin: string, position: number];
178
+ export type LockDependencySet = readonly LockDependency[];
179
+
180
+ export function compareTokens(a: FencingToken, b: FencingToken): number {
181
+ return a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
182
+ }
183
+
184
+ function isFencingToken(value: unknown): value is FencingToken {
185
+ return (
186
+ Array.isArray(value) &&
187
+ value.length === 3 &&
188
+ value.every((part) => typeof part === 'number' && Number.isFinite(part))
189
+ );
190
+ }
191
+
192
+ export interface LockReleaseEntry {
193
+ type: 'lockRelease';
194
+ /** The locked record's id. Control entries carry it here, never as the audit entry's recordId. */
195
+ key: any;
196
+ /** Node that held the delegation being released. */
197
+ requester: string;
198
+ /**
199
+ * The released delegation's whole fencing token. The counter alone is NOT enough to identify it: a
200
+ * home that restarts, or whose coordinator is recreated, begins counting again, so a delayed
201
+ * release from a previous incarnation would match a live grant's counter and clear it while its
202
+ * delegate is still admitting.
203
+ */
204
+ token: FencingToken;
205
+ /**
206
+ * Inherited clean-handoff lineage. Null hands back an unused first grant without advancing it; a
207
+ * renewed grant still drops retained lineage because its earlier token may have admitted writes.
208
+ * Absent means legacy/unknown.
209
+ */
210
+ dependencies?: LockDependencySet | null;
211
+ }
212
+
213
+ /**
214
+ * The §7.2 recovery fence. It names no key and no token: it is appended after every transaction the
215
+ * writing node had committed when it was requested, so a peer that has applied that origin's log
216
+ * through this entry has applied all of them. The coordinator never acts on one.
217
+ */
218
+ export interface LockBarrierEntry {
219
+ type: 'lockBarrier';
220
+ /** Supplied by the requesting transport; distinguishes barriers an origin stamped identically across a restart. */
221
+ nonce: number;
222
+ }
223
+
224
+ export type LockControlEntry = LockReleaseEntry | LockBarrierEntry;
225
+
226
+ /**
227
+ * What a home replies to a delegation request. One shape rather than a discriminated union: the
228
+ * repo compiles with `strict: false`, where TypeScript does not narrow a union on a boolean literal,
229
+ * so a union here would type-check the denial fields as absent on every branch and then not enforce
230
+ * it. `granted` says which half is populated.
231
+ */
232
+ export interface DelegationReply {
233
+ granted: boolean;
234
+ /** Granted only. */
235
+ token?: FencingToken;
236
+ /** Granted only. How long the delegate may admit for, as a DURATION — never a remote clock reading. */
237
+ leaseMs?: number;
238
+ /** Granted only. Null selects the recovery barrier; an array is the exact clean-handoff fence. */
239
+ dependencies?: LockDependencySet | null;
240
+ /**
241
+ * Denied only. `contended` is the one reason that means another node holds the key, and so the only
242
+ * one an exhausted wait may report as 423. `generation` (the two sides hold different home maps),
243
+ * `unknown-node` (this node is not named in the map) and `quarantine` (the home is inside its §4.3
244
+ * restart interval) all describe something other than contention, so each ends as a retryable 503
245
+ * rather than telling the caller a key nobody holds is held. `timeout` is not a home answer at all
246
+ * — it is this node's own deadline ending its probe — and never classifies a wait (DESIGN.md).
247
+ */
248
+ reason?: 'contended' | 'generation' | 'unknown-node' | 'capacity' | 'not-home' | 'quarantine' | 'timeout';
249
+ /** Denied with `generation`, so a stale requester can re-derive the ring without another round trip. */
250
+ generation?: number;
251
+ retryAfterMs?: number;
252
+ }
253
+
254
+ /** A completed home reply, bound to the route that produced it so a ring change retires it. */
255
+ interface LastCompletedReply {
256
+ reply: DelegationReply;
257
+ home: string;
258
+ generation: number;
259
+ }
260
+
261
+ export interface DelegationRequest {
262
+ key: any;
263
+ /** The asking node. Established by the transport, never read from an untrusted payload. */
264
+ requester: string;
265
+ generation: number;
266
+ leaseMs: number;
267
+ }
268
+
269
+ export interface DelegationRecall {
270
+ key: any;
271
+ token: FencingToken;
272
+ }
273
+
274
+ /**
275
+ * Supplied by harper-pro. Core never computes cluster topology; it only refuses to promise a
276
+ * cluster-wide lock that this contract cannot back.
277
+ */
278
+ export interface ClusterLockTransport {
279
+ /**
280
+ * The operator-agreed home map for the database, or undefined while none is available — before the
281
+ * node has the current generation, or while peers disagree about its digest. Core fails closed on
282
+ * undefined rather than guessing a ring.
283
+ *
284
+ * **One obligation core cannot check, and relies on** (§4.3): a generation change is
285
+ * operator-sequenced, so `g+1` may only be answered here once the control plane's one-shot
286
+ * activation record is active — every old home quiesced (each acknowledgement bound to the
287
+ * acknowledger's `homeIncarnation`, so a restart during the drain invalidates it) or externally
288
+ * fenced, then `DELEGATION_LEASE_MS + skew` elapsed. Core cannot observe what happened on other
289
+ * nodes; the operator can.
290
+ *
291
+ * The other interval — a restart of *this* process — is core's own, because a generation does not
292
+ * advance on a restart and there is no external event to hang it on. See `#grantableAfterMono`.
293
+ */
294
+ homeMap(database: string): LockHomeMap | undefined;
295
+ /**
296
+ * Overrides core's cold-start grant quarantine (§4.3). Set it only where a previous incarnation
297
+ * of this process provably issued nothing — a fresh database, a first start, or a test. Because the
298
+ * same attestation enables the virgin-key freshness fast path, it must prove that no earlier
299
+ * incarnation delegated any key under the current home-map generation at any time, not merely that
300
+ * its last delegation lease has elapsed. Omitted means core enforces the full
301
+ * `DELEGATION_LEASE_MS + skew` from that coordinator's construction — see `#grantableAfterMono` for
302
+ * why neither process start nor thread start is a sound anchor.
303
+ */
304
+ grantableAfterMono?: number;
305
+ /**
306
+ * Whether this worker thread owns lock coordination for the process. Coordinator state is
307
+ * per-thread while the key lock it arbitrates is process-wide, so a second thread running its own
308
+ * delegations would arbitrate against a different view. Core fails closed off the owner thread.
309
+ */
310
+ ownsCoordination(): boolean;
311
+ /** Ask `node` (the key's home) for a delegation. Unicast; rejects if the node is unreachable. */
312
+ requestDelegation(
313
+ node: string,
314
+ database: string,
315
+ table: string,
316
+ request: DelegationRequest
317
+ ): Promise<DelegationReply>;
318
+ /** Home → delegate. Resolves once the delegate has drained and stopped admitting. */
319
+ recallDelegation(node: string, database: string, table: string, recall: DelegationRecall): Promise<void>;
320
+ /**
321
+ * Obtain an admission from the worker thread that coordinates this database, for a `lock()` served
322
+ * on a thread that is not the owner. Present only when the transport can relay across threads
323
+ * (harper-pro#852); a transport without it makes `acquire()` fail closed off the owner thread, as
324
+ * before. The returned `LockRound` was minted by the owner's coordinator; the caller's coordinator
325
+ * records it as a REMOTE admission and installs the handle's revoker locally, so a recall on the
326
+ * owner fences a write this thread's handle staged. `mintedMono` is comparable across threads
327
+ * because `performance.now()` shares one time origin process-wide.
328
+ */
329
+ acquireOnOwner?(database: string, table: string, key: any, leaseMs: number, waitMs: number): Promise<LockRound>;
330
+ /** Release a remote admission on the owner thread (the counterpart of `acquireOnOwner`). */
331
+ releaseOnOwner?(database: string, table: string, key: any, admissionId: number): Promise<void> | void;
332
+ /**
333
+ * Establish an exact clean-handoff dependency set, or recover the strongest reachable-member
334
+ * barrier when `dependencies` is null. Recovery returns the captured positions for current lock
335
+ * participants that were made visible; it may drain additional replication peers without carrying
336
+ * them in the key's lineage. Clean waits may return void. Concurrent recovery snapshots should be
337
+ * coalesced.
338
+ *
339
+ * Recovery asks each reachable member for a `lockBarrier` entry (`writeLockBarrier`) and drains
340
+ * that member's stream through the position it returns. Core races this promise against the
341
+ * lock's own deadline but cannot cancel it; `deadlineMs` is the wait remaining at the call, so the
342
+ * transport can bound its own work to it instead of outliving the lock that asked.
343
+ */
344
+ establishLockFreshness(
345
+ database: string,
346
+ table: string,
347
+ key: any,
348
+ dependencies: LockDependencySet | null,
349
+ deadlineMs: number
350
+ ): Promise<LockDependencySet | void>;
351
+ /** Emit a control entry and return the committed entry's local origin-log position when available. */
352
+ writeControl?(table: string, entry: LockControlEntry): Promise<number | void> | number | void;
353
+ /**
354
+ * Assigned at registration so a transport can push a received entry in directly. `author` and
355
+ * `position` come from the authenticated origin-log header, never from the payload.
356
+ */
357
+ onControlEntry?(database: string, table: string, entry: LockControlEntry, author: string, position: number): void;
358
+ /** Assigned at registration. Inbound delegation request from a peer, for a key this node homes. */
359
+ onDelegationRequest?(database: string, table: string, request: DelegationRequest): Promise<DelegationReply>;
360
+ /** Assigned at registration. Inbound recall from a key's home. */
361
+ onDelegationRecall?(database: string, table: string, recall: DelegationRecall): Promise<void>;
362
+ }
363
+
364
+ // A private structure dictionary, so a control payload can never contribute to — or depend on — the
365
+ // table's own, and never passes through schema projection (see writeLockControlEntry in Table.ts).
366
+ // Record mode stays ON deliberately: the reader is the receiving table's decoder (via
367
+ // `auditRecord.getValue`), which repurposes a range of positive fixints as structure ids. A payload
368
+ // packed without record mode writes an integer record key of 64..127 as a bare fixint, which that
369
+ // decoder then reads as a structure header and the whole entry fails to decode.
370
+ let controlStructures: unknown[] = [];
371
+ let controlPackr = new Packr({ structures: controlStructures });
372
+
373
+ export function encodeLockControlPayload(entry: LockControlEntry): Uint8Array {
374
+ if (entry.type === 'lockBarrier') return controlPackr.pack([1, entry.nonce]);
375
+ const [generation, homeIncarnation, counter] = entry.token;
376
+ if (entry.dependencies === undefined)
377
+ return controlPackr.pack([entry.key, entry.requester, generation, homeIncarnation, counter]);
378
+ return controlPackr.pack([1, entry.key, entry.requester, generation, homeIncarnation, counter, entry.dependencies]);
379
+ }
380
+
381
+ function isNodeName(value: unknown): value is string {
382
+ return typeof value === 'string' && value.length > 0 && value.length <= MAX_NODE_NAME_LENGTH;
383
+ }
384
+
385
+ function isDuration(value: unknown, min: number, max: number): value is number {
386
+ return typeof value === 'number' && Number.isFinite(value) && value >= min && value <= max;
387
+ }
388
+
389
+ /**
390
+ * Accept exactly what `ordered-binary` encodes, because that is what `keyIdOf` runs on the key.
391
+ * Refusing a shape the encoder handles makes the home answer `not-home` for a key nobody holds, and
392
+ * the requester retries that to its own 423. A `Uint8Array` returns from the unpack as a `Buffer`
393
+ * and a small `bigint` as a `number`; both encode to the same stored key, which is the identity at
394
+ * issue rather than the JS value.
395
+ */
396
+ function isEncodableKey(value: unknown): boolean {
397
+ const type = typeof value;
398
+ if (type === 'string' || type === 'bigint' || type === 'number' || type === 'boolean') return true;
399
+ if (value === null || value instanceof Uint8Array) return true;
400
+ return Array.isArray(value) && (value as unknown[]).every(isEncodableKey);
401
+ }
402
+
403
+ /**
404
+ * Decode a received control payload, or undefined when it is not one this version understands. The
405
+ * tuple length is validated exactly: a future version that grows the payload must bump the type
406
+ * rather than widen this one, since a partially-understood release would clear a delegation on terms
407
+ * the sender did not intend.
408
+ */
409
+ export function decodeLockControlPayload(type: unknown, value: unknown): LockControlEntry | undefined {
410
+ if (type !== 'lockRelease' && type !== 'lockBarrier') return undefined;
411
+ let tuple: unknown;
412
+ try {
413
+ tuple = value instanceof Uint8Array ? controlPackr.unpack(value) : value;
414
+ } catch {
415
+ return undefined;
416
+ }
417
+ if (!Array.isArray(tuple)) return undefined;
418
+ if (type === 'lockBarrier') {
419
+ if (tuple.length !== 2 || tuple[0] !== 1) return undefined;
420
+ const nonce = tuple[1];
421
+ return typeof nonce === 'number' && Number.isFinite(nonce) ? { type: 'lockBarrier', nonce } : undefined;
422
+ }
423
+ if (tuple.length !== 5 && tuple.length !== 7) return undefined;
424
+ try {
425
+ return decodeTuple(tuple);
426
+ } catch {
427
+ // isEncodableKey recurses; a deeply nested array in a peer or replayed payload would otherwise
428
+ // raise a RangeError into the replicated apply loop instead of being dropped as malformed.
429
+ return undefined;
430
+ }
431
+ }
432
+
433
+ function decodeTuple(tuple: unknown[]): LockReleaseEntry | undefined {
434
+ const versioned = tuple.length === 7;
435
+ if (versioned && tuple[0] !== 1) return undefined;
436
+ const offset = versioned ? 1 : 0;
437
+ const [key, requester, generation, homeIncarnation, counter] = tuple.slice(offset, offset + 5);
438
+ if (!isEncodableKey(key) || !isNodeName(requester)) return undefined;
439
+ for (const part of [generation, homeIncarnation, counter])
440
+ if (typeof part !== 'number' || !Number.isFinite(part)) return undefined;
441
+ let dependencies: LockDependencySet | null | undefined;
442
+ if (versioned) {
443
+ const rawDependencies = tuple[6];
444
+ if (rawDependencies === null) dependencies = null;
445
+ else dependencies = normalizeDependencies(rawDependencies);
446
+ }
447
+ const entry: LockReleaseEntry = {
448
+ type: 'lockRelease',
449
+ key,
450
+ requester,
451
+ token: [generation, homeIncarnation, counter] as FencingToken,
452
+ };
453
+ if (versioned) entry.dependencies = dependencies;
454
+ return entry;
455
+ }
456
+
457
+ function normalizeDependencies(value: unknown, homes?: readonly string[]): LockDependencySet | undefined {
458
+ if (!Array.isArray(value) || value.length > MAX_LOCK_DEPENDENCIES) return undefined;
459
+ const positions = new Map<string, number>();
460
+ const allowedOrigins = homes && new Set(homes);
461
+ for (const dependency of value) {
462
+ if (!Array.isArray(dependency) || dependency.length !== 2) return undefined;
463
+ const [origin, position] = dependency;
464
+ if (!isNodeName(origin) || typeof position !== 'number' || !Number.isFinite(position) || position < 0)
465
+ return undefined;
466
+ if (allowedOrigins && !allowedOrigins.has(origin)) return undefined;
467
+ if (positions.has(origin)) return undefined;
468
+ positions.set(origin, position);
469
+ }
470
+ return [...positions].sort(([a], [b]) => a.localeCompare(b));
471
+ }
472
+
473
+ class DelegatedKeyFilter {
474
+ #bits = new Uint32Array(DELEGATED_KEY_FILTER_WORDS);
475
+
476
+ add(key: unknown): void {
477
+ const [first, second] = this.#hashes(key);
478
+ for (let index = 0; index < 4; index++) this.#set((first + Math.imul(index, second)) >>> 0);
479
+ }
480
+
481
+ has(key: unknown): boolean {
482
+ const [first, second] = this.#hashes(key);
483
+ for (let index = 0; index < 4; index++) if (!this.#get((first + Math.imul(index, second)) >>> 0)) return false;
484
+ return true;
485
+ }
486
+
487
+ clear(): void {
488
+ this.#bits.fill(0);
489
+ }
490
+
491
+ copyFrom(other: DelegatedKeyFilter): void {
492
+ this.#bits.set(other.#bits);
493
+ }
494
+
495
+ #set(hash: number): void {
496
+ const bit = hash % (this.#bits.length * 32);
497
+ this.#bits[bit >>> 5] |= 1 << (bit & 31);
498
+ }
499
+
500
+ #get(hash: number): boolean {
501
+ const bit = hash % (this.#bits.length * 32);
502
+ return (this.#bits[bit >>> 5] & (1 << (bit & 31))) !== 0;
503
+ }
504
+
505
+ #hashes(key: unknown): [number, number] {
506
+ const value = `${typeof key}:${String(key)}`;
507
+ let first = 0x811c9dc5;
508
+ let second = 0x9e3779b9;
509
+ for (let index = 0; index < value.length; index++) {
510
+ first = Math.imul(first ^ value.charCodeAt(index), 0x01000193);
511
+ second = Math.imul(second ^ value.charCodeAt(index), 0x85ebca6b);
512
+ }
513
+ return [first >>> 0, (second | 1) >>> 0];
514
+ }
515
+ }
516
+
517
+ /**
518
+ * Rendezvous (highest-random-weight) hash: a key's home is the node with the greatest score for that
519
+ * key. Chosen over a modulo of a key hash because a generation change moves only the keys homed on a
520
+ * departing node, rather than re-homing the whole space — which matters because every re-homed key
521
+ * pays the §7.2 recovery path on its next lock.
522
+ *
523
+ * The hash is FNV-1a over the member name and the key's stable id. It does not need to be
524
+ * cryptographic: it is not a defense against anything, only a deterministic agreement between nodes
525
+ * that already agree on `homes`.
526
+ */
527
+ function scoreFor(member: string, keyId: string): number {
528
+ let hash = 0x811c9dc5;
529
+ for (let i = 0; i < member.length; i++) {
530
+ hash ^= member.charCodeAt(i);
531
+ hash = Math.imul(hash, 0x01000193);
532
+ }
533
+ // A separator that cannot appear in either operand, so ("ab","c") and ("a","bc") cannot collide.
534
+ hash ^= 0xff;
535
+ hash = Math.imul(hash, 0x01000193);
536
+ for (let i = 0; i < keyId.length; i++) {
537
+ hash ^= keyId.charCodeAt(i);
538
+ hash = Math.imul(hash, 0x01000193);
539
+ }
540
+ return hash >>> 0;
541
+ }
542
+
543
+ /**
544
+ * The string the ring hashes for a key. §4.4 scopes it by database and table: hashing the record id
545
+ * alone would home id `42` in every table of every database on the same node, concentrating unrelated
546
+ * hot keys on one arbiter. The separator cannot appear in a name or a stringified key id.
547
+ */
548
+ export function ringKeyFor(database: string, table: string, keyId: unknown): string {
549
+ return `${database}\u0000${table}\u0000${String(keyId)}`;
550
+ }
551
+
552
+ export function homeFor(keyId: string, homes: string[]): string | undefined {
553
+ let best: string | undefined;
554
+ let bestScore = -1;
555
+ for (const home of homes) {
556
+ const score = scoreFor(home, keyId);
557
+ // Ties break on the name so every node picks the same home from the same set.
558
+ if (score > bestScore || (score === bestScore && best !== undefined && home > best)) {
559
+ bestScore = score;
560
+ best = home;
561
+ }
562
+ }
563
+ return best;
564
+ }
565
+
566
+ /** A delegation this node holds: the right to admit critical sections on one key. */
567
+ interface Delegation {
568
+ key: any;
569
+ token: FencingToken;
570
+ dependencies: LockDependencySet;
571
+ /** Monotonic deadline on THIS node. A delegate stops admitting here. */
572
+ expiresMono: number;
573
+ /** Set by a recall. No new admission may start, but live ones are drained first. */
574
+ recalled: boolean;
575
+ /**
576
+ * Every admission this delegation is still answerable for, by id. An entry stays here after its
577
+ * caller unlocks: §6 revokes CAPABILITY, not admission. A caller that staged a write and then
578
+ * called `unlock()` has nothing left to wait on, but its write is still uncommitted and would land
579
+ * after the successor was admitted — so surrender has to fence it, which means keeping its revoker
580
+ * until its own lease runs out or authority is lost.
581
+ *
582
+ * Addressed by id through `#admissions` rather than owned by the delegation OBJECT, so a handle
583
+ * survives a renewal with its delegation and is revoked with it when authority is actually lost.
584
+ */
585
+ admissions: Map<number, Admission>;
586
+ /**
587
+ * How many of those admissions have not unlocked yet. A drain waits for THIS to reach zero — an
588
+ * unlocked-but-staged write is revoked rather than waited for (harper#2580).
589
+ */
590
+ holding: number;
591
+ /** `admissions.size` at which the next full expiry sweep runs; see `#pruneAdmissions`. */
592
+ sweepAtSize: number;
593
+ /** Resolvers waiting for the drain to finish, so recall can reply once rather than poll. */
594
+ drained?: (() => void)[];
595
+ /** The one in-flight `#surrender` for this delegation; see there for why it is memoized. */
596
+ surrendering?: Promise<void>;
597
+ }
598
+
599
+ /**
600
+ * One `lock()` this thread admitted from the OWNER thread (harper-pro#852), addressed by a LOCAL id
601
+ * distinct from the owner's own admission id (see `#remoteByOwnerId`), so a stale local admission can
602
+ * never share a number with an incoming owner-minted one. `revoke` fences this thread's handle;
603
+ * `revoked` latches a revoke that landed before `registerAdmission` supplied the real revoker.
604
+ * `fenceWaiters` resolve once the handle is provably fenced — the owner waits on them before it writes
605
+ * the release, so an ack never claims a fence the handle has not actually taken.
606
+ */
607
+ interface RemoteAdmission {
608
+ /** The owner's admission id, echoed on release and matched by an inbound revoke. */
609
+ ownerAdmissionId: number;
610
+ key: any;
611
+ /** The caller-side handle fence — today always `Table.lock`'s `() => handle.revokeLease()`, which is
612
+ * synchronous and total, never the owner-side async relay revoker (that lives on a delegation
613
+ * `Admission` instead). `registerAdmission` accepts an async revoker all the same, so the ack paths
614
+ * here settle on the outcome rather than assuming this one. */
615
+ revoke: () => void;
616
+ /** Monotonic deadline of the handle's own lease; the entry is dropped once past it. */
617
+ expiresMono: number;
618
+ revoked: boolean;
619
+ fenceWaiters: (() => void)[];
620
+ }
621
+
622
+ /** One `lock()` admitted under a delegation. */
623
+ interface Admission {
624
+ /**
625
+ * Fences the handle's write capability. A no-op until `registerAdmission` supplies the real one. A
626
+ * handle admitted on another worker thread revokes over a message and resolves once that thread has
627
+ * fenced it; `#surrender` waits for that (or the admission's own lease) before writing the release.
628
+ */
629
+ revoke: () => void | Promise<void>;
630
+ /** Monotonic deadline of the handle's OWN lease, after which it fences itself and can be dropped. */
631
+ expiresMono: number;
632
+ /** False once the caller unlocked: it no longer blocks a drain, but is still revocable. */
633
+ holding: boolean;
634
+ }
635
+
636
+ /** A delegation this node issued as a key's home. */
637
+ interface HomeGrant {
638
+ key: any;
639
+ delegate: string;
640
+ token: FencingToken;
641
+ /** Requirement handed to this delegate; reused if the same node renews after losing local state. */
642
+ dependencies: LockDependencySet | null;
643
+ /** This grant advanced from an earlier token that may already have admitted writes. */
644
+ renewed?: boolean;
645
+ /**
646
+ * Monotonic deadline on THIS node, set to the delegate's lease PLUS skew. The home always outwaits
647
+ * the delegate, so it can never re-grant a key the previous delegate still believes it holds.
648
+ */
649
+ expiresMono: number;
650
+ recalling?: Promise<void>;
651
+ /** Set once the delegate confirmed it stopped admitting, so the recall is never re-sent. */
652
+ recallConfirmed?: boolean;
653
+ /** After a FAILED recall, the earliest this home may try that delegate again (`RECALL_RETRY_MS`). */
654
+ recallRetryAfterMono?: number;
655
+ }
656
+
657
+ interface PendingDelegation {
658
+ key: any;
659
+ token: FencingToken;
660
+ recalled: boolean;
661
+ recalledPromise: Promise<void>;
662
+ markRecalled: () => void;
663
+ }
664
+
665
+ const PENDING_DELEGATION_RECALLED = Symbol('pending delegation recalled');
666
+
667
+ function createPendingDelegation(key: any, token: FencingToken, recalledBeforeReply: boolean): PendingDelegation {
668
+ let resolveRecall: () => void;
669
+ const pending: PendingDelegation = {
670
+ key,
671
+ token,
672
+ recalled: false,
673
+ recalledPromise: new Promise((resolve) => (resolveRecall = resolve)),
674
+ markRecalled() {
675
+ if (pending.recalled) return;
676
+ pending.recalled = true;
677
+ resolveRecall();
678
+ },
679
+ };
680
+ if (recalledBeforeReply) pending.markRecalled();
681
+ return pending;
682
+ }
683
+
684
+ interface PendingRequest {
685
+ recalledToken?: FencingToken;
686
+ }
687
+
688
+ export interface LockRound {
689
+ /** The identity to stamp the holder's writes with. */
690
+ tsR: number;
691
+ /** The monotonic reading the admission's lease is measured from. */
692
+ mintedMono: number;
693
+ /**
694
+ * Identifies THIS admission for the life of the handle it produced. The caller hands it back on
695
+ * registration and on release. An id rather than the delegation's token, because the token changes
696
+ * on renewal while the admission does not: addressing by token made a renewed delegation lose
697
+ * track of handles it was still responsible for.
698
+ */
699
+ admissionId: number;
700
+ }
701
+
702
+ export interface LockCoordinatorOptions {
703
+ database: string;
704
+ table: string;
705
+ /** This node's globally stable name. */
706
+ nodeId: string;
707
+ transport: ClusterLockTransport;
708
+ /**
709
+ * Emit one control entry. Core passes the transport's own `writeControl` when it has one and its
710
+ * transaction-log writer otherwise, since writing to the local log IS the send.
711
+ */
712
+ writeControl: (entry: LockControlEntry) => Promise<number | void> | number | void;
713
+ /** Stable map key for a record id; core passes `writeKeyId`. */
714
+ keyIdOf: (key: any) => unknown;
715
+ /** Mints the holder's stamp; core passes the primary store's monotonic timestamp. */
716
+ nextTimestamp: () => number;
717
+ /** Monotonic clock. Every expiry decision is made on this. */
718
+ monotonic?: () => number;
719
+ skewMs?: number;
720
+ /**
721
+ * The coordinator this one replaces, when a transport is re-registered (a component reload is
722
+ * enough). Its live authority is MOVED here rather than discarded: the transport changed, but the
723
+ * handles it admitted did not, and a successor that started with an empty grant table could hand
724
+ * the same key to another node with no lease time elapsed. That is the "a home may never forget a
725
+ * grant before its expiry" rule (§8) applied across the swap rather than only within one
726
+ * coordinator's life. §11 of the design note calls for exactly this — carry live authority across,
727
+ * or fence and settle every outstanding handle before granting; carrying it across costs nothing.
728
+ */
729
+ adopt?: LockCoordinator;
730
+ /**
731
+ * Overrides the cold-start grant quarantine (§4.3). Pass `-Infinity` only where a previous
732
+ * incarnation provably issued nothing — a fresh database, or a test. Required when `monotonic` is
733
+ * an injected clock the test drives itself. See `#grantableAfterMono`.
734
+ */
735
+ grantableAfterMono?: number;
736
+ /** False in tests, which drive `tick()` themselves. */
737
+ autoTick?: boolean;
738
+ }
739
+
740
+ /**
741
+ * What a coordinator closed WITHOUT a successor leaves for its eventual replacement, per table.
742
+ *
743
+ * `close()` drops the grant table, but the delegations those grants authorize are still live on other
744
+ * nodes until their own deadlines — closing is a local event that no peer observes. A replacement
745
+ * built before then must not grant those keys again, and must not restart the counter into tokens the
746
+ * closed coordinator already issued. `#grantableAfterMono` covers the same hazard across a process
747
+ * RESTART; this covers it across an unregister and re-register inside one process, which a
748
+ * process-start reading cannot see.
749
+ */
750
+ const retiredCoordinators = new Map<string, { grantableAfterMono: number; counter: number }>();
751
+
752
+ /**
753
+ * The highest home-map generation this thread has acted under, per DATABASE — the scope the generation
754
+ * itself has, not the per-table scope a token is compared at. A generation is the high-order component
755
+ * of every fencing token (§5.1), so going backwards re-mints tokens that order BELOW ones already
756
+ * handed out, and a delayed write under the newer generation then defeats its successor. Refusing it
757
+ * database-wide is strictly stronger than refusing it per table and costs nothing: one rolled-back
758
+ * publish is one event. The map is operator-published, so the rollback route is a configuration
759
+ * restore or a partial publish rather than a protocol bug — which is why it is refused here rather
760
+ * than assumed away. Remembering it across a restart, and across threads, is harper-pro's half.
761
+ */
762
+ const highestGeneration = new Map<string, number>();
763
+
764
+ /** Bound one drain step so a single unresponsive delegate cannot consume the whole transition budget. */
765
+ function withDeadline<T>(work: Promise<T>, ms: number, message = 'the quiesce deadline elapsed'): Promise<T> {
766
+ if (!(ms > 0)) return Promise.reject(new Error(message));
767
+ const timer = delay(ms);
768
+ work.then(
769
+ () => timer.cancel(),
770
+ () => timer.cancel()
771
+ );
772
+ return Promise.race([
773
+ work,
774
+ timer.promise.then<T>(() => {
775
+ throw new Error(message);
776
+ }),
777
+ ]);
778
+ }
779
+
780
+ const tickingCoordinators = new Set<LockCoordinator>();
781
+ /**
782
+ * Every coordinator alive on this thread, so a membership transition can quiesce a whole database
783
+ * rather than one table at a time (harper-pro#856). The per-`(database, table)` resolvers cannot
784
+ * enumerate: they answer a name you already have.
785
+ */
786
+ const liveCoordinators = new Set<LockCoordinator>();
787
+ let tickTimer: ReturnType<typeof setInterval> | undefined;
788
+ function ensureTicking() {
789
+ if (tickTimer || tickingCoordinators.size === 0) return;
790
+ tickTimer = setInterval(() => {
791
+ for (const coordinator of tickingCoordinators) {
792
+ try {
793
+ coordinator.tick();
794
+ } catch (error) {
795
+ warnOnce('lock coordinator tick failed', error);
796
+ }
797
+ }
798
+ if (tickingCoordinators.size === 0) {
799
+ clearInterval(tickTimer);
800
+ tickTimer = undefined;
801
+ }
802
+ }, TICK_INTERVAL_MS);
803
+ tickTimer.unref?.();
804
+ }
805
+
806
+ /** Stands in until `registerAdmission` supplies the handle's real revoker. */
807
+ function noRevoke() {}
808
+
809
+ /**
810
+ * Fire a revoker in a fire-and-forget context, absorbing both a synchronous throw and a rejected
811
+ * promise. A relayed revoker is `() => Promise<void>` and may reject on a dead sibling port; a
812
+ * discarded rejection would exit the worker under Node's default policy, taking every coordinator on
813
+ * the thread. Callers that must WAIT for the fence (`#revokeAllAndSettle`, `revokeRemoteAdmission`)
814
+ * handle the outcome themselves and do not use this.
815
+ */
816
+ function isPromiseLike(value: unknown): value is Promise<unknown> {
817
+ return value != null && typeof (value as Promise<unknown>).then === 'function';
818
+ }
819
+ function fireRevokeAndForget(revoke: () => void | Promise<void>): void {
820
+ try {
821
+ const outcome = revoke();
822
+ if (isPromiseLike(outcome))
823
+ outcome.catch((error) => warnOnce('a fire-and-forget record lock revoke failed', error));
824
+ } catch (error) {
825
+ warnOnce('a fire-and-forget record lock revoke threw', error);
826
+ }
827
+ }
828
+
829
+ /**
830
+ * Rate-limit rather than latch. These messages report a transport, writer or revoker that failed, and
831
+ * a latch for the life of the process would show an operator the first occurrence and then hide a
832
+ * fault that persists for days. One per message per window is enough to keep a hot loop from flooding
833
+ * the log while still showing that the condition is ongoing.
834
+ */
835
+ const warnedMessages = new Map<string, number>();
836
+ function warnOnce(message: string, detail?: unknown) {
837
+ const now = performance.now();
838
+ const last = warnedMessages.get(message);
839
+ if (last !== undefined && now - last < WARN_INTERVAL_MS) return;
840
+ warnedMessages.set(message, now);
841
+ harperLogger.warn?.(message, detail);
842
+ }
843
+
844
+ export class LockCoordinator {
845
+ readonly database: string;
846
+ readonly table: string;
847
+ readonly nodeId: string;
848
+ readonly transport: ClusterLockTransport;
849
+ #writeControl: (entry: LockControlEntry) => Promise<number | void> | number | void;
850
+ #keyIdOf: (key: any) => unknown;
851
+ #nextTimestamp: () => number;
852
+ #monotonic: () => number;
853
+ #skewMs: number;
854
+ #autoTick: boolean;
855
+ /** Whoever this coordinator's authority was handed to, for replies that land after the swap. */
856
+ #successor: LockCoordinator | undefined;
857
+ /** Set when this coordinator's state was moved to a successor, so `close()` must not expire it. */
858
+ #handedOff = false;
859
+ /**
860
+ * Monotonic reading before which this coordinator may not grant as a home — the §4.3 restart
861
+ * quarantine, and core's own to enforce.
862
+ *
863
+ * A coordinator that started cold has no record of the delegations a previous incarnation issued,
864
+ * and those can still be admitting on their holders. Nothing external bounds them: the home map is
865
+ * immutable, so its generation does not advance merely because a process or a worker restarted. The
866
+ * only instant core can prove nothing else was granting under is this coordinator's own
867
+ * construction, so the quarantine runs `DELEGATION_LEASE_MS + skew` from there — by which point
868
+ * every delegation a previous incarnation could have issued has expired. It costs availability on
869
+ * this node's own share of the ring and nothing elsewhere: keys homed on other nodes are acquired
870
+ * immediately, and `adopt` plus the retirement record waive it wherever a predecessor's authority
871
+ * is actually known.
872
+ *
873
+ * A deployment that can prove a previous incarnation issued nothing overrides it through
874
+ * `ClusterLockTransport.grantableAfterMono`. Generation CHANGES need nothing here: the
875
+ * delegate-side generation check in `#liveDelegation` is what stops the old delegate.
876
+ */
877
+ #grantableAfterMono: number;
878
+ /**
879
+ * How long until this coordinator can rule out authority issued before it took over — 0 when it
880
+ * already can. Non-mutating, unlike `#ownershipHorizon`, which records ownership as a side effect.
881
+ *
882
+ * The waiver is deliberately NOT consulted. `grantableAfterMono` attests that no previous
883
+ * INCARNATION OF THIS PROCESS delegated; it says nothing about a sibling thread that was
884
+ * coordinating until this instant, and a coordinator built while already owning keeps the waiver
885
+ * without ever observing that handoff — which let a takeover worker in a first-incarnation process
886
+ * report a clean drain while the previous owner's delegates were still admitting.
887
+ *
888
+ * The cost is that a freshly built coordinator cannot prove quiescence for a full lease. That falls
889
+ * only on the case that does not need the proof: a node with no delegations yet is bootstrapping
890
+ * generation 1, where there is nothing to drain and no interval to skip.
891
+ */
892
+ unprovenOwnershipMs(): number {
893
+ const horizon = DELEGATION_LEASE_MS + this.#skewMs;
894
+ if (this.#ownedSinceMono === undefined) return horizon;
895
+ return Math.max(0, this.#ownedSinceMono + horizon - this.#monotonic());
896
+ }
897
+ /**
898
+ * When this coordinator was last observed to own coordination, and `undefined` while it does not.
899
+ *
900
+ * The quarantine has to run from here and not only from construction: a coordinator is built when a
901
+ * transport registers, but `ownsCoordination()` can flip to true long afterwards — a thread that
902
+ * took over from an owner that died. By then the construction horizon has aged out, and this
903
+ * coordinator would grant immediately over delegations the previous OWNER issued. Regaining
904
+ * ownership re-arms it for the same reason: something else was coordinating in between.
905
+ */
906
+ #ownedSinceMono: number | undefined;
907
+ /** Rate-limits `tick()`'s ownership poll to `OWNERSHIP_POLL_MS`. */
908
+ #lastOwnershipPollMono = -Infinity;
909
+ /**
910
+ * The `homeIncarnation` this coordinator has been coordinating under. A different one is the
911
+ * transport saying another coordination incarnation ran for this node (§5.1), which is the only
912
+ * statement about CONTINUITY available here — see `#ownershipHorizon`.
913
+ */
914
+ #coordinatingIncarnation: number | undefined;
915
+ /**
916
+ * Set when the caller supplied an explicit horizon. It waives both halves of the quarantine only
917
+ * for this coordinator's FIRST ownership interval, and `#ownershipHorizon` clears it at the first
918
+ * observed gap: the attestation behind it is "no previous incarnation of this process issued
919
+ * anything", which is a claim about process start and not about a sibling thread that coordinated
920
+ * while this one did not.
921
+ */
922
+ #quarantineWaived: boolean;
923
+ /** Keys this node holds a delegation for. */
924
+ #delegations = new Map<unknown, Delegation>();
925
+ /** Grants received but not yet installed because their freshness barrier is still running. */
926
+ #pendingDelegations = new Map<unknown, PendingDelegation>();
927
+ /** Outbound requests whose grant token is not known yet, so an early recall cannot be lost. */
928
+ #pendingRequests = new Map<unknown, PendingRequest>();
929
+ /**
930
+ * Every live admission, by id, and the delegation answerable for it. Coordinator-level rather than
931
+ * per-delegation so a release can find its admission after the delegation was renewed or replaced.
932
+ */
933
+ #admissions = new Map<number, Delegation>();
934
+ #nextAdmissionId = 1;
935
+ /**
936
+ * Admissions this thread obtained from the OWNER thread (harper-pro#852), keyed by a LOCAL id drawn
937
+ * from `#nextAdmissionId` — never the owner's id, so it cannot collide with a live local admission.
938
+ * This thread holds the handle (native key, staged writes, lease timer) while the delegation that
939
+ * authorizes it lives on the owner. Allocated lazily so a coordinator that never serves an off-owner
940
+ * lock pays nothing. An entry is kept after the caller unlocks, revoker included, until the handle's
941
+ * own lease runs out (owner-side §6 retention: an unlocked-but-staged write is still fenceable).
942
+ */
943
+ #remoteAdmissions: Map<number, RemoteAdmission> | undefined;
944
+ /** Owner admission id → this thread's local id, so an inbound revoke (which names the owner id)
945
+ * reaches the right entry. */
946
+ #remoteByOwnerId: Map<number, number> | undefined;
947
+ /** Owner admission ids whose revoke arrived before `#acquireFromOwner` installed the entry, with the
948
+ * ack resolvers waiting on the eventual fence and the monotonic time they arrived. Drained when the
949
+ * entry installs (`#acquireFromOwner`); an entry whose acquire never lands (a revoke for an admission
950
+ * this thread already dropped) is resolved and swept once its wait exceeds a lease (`tick`). */
951
+ #pendingRemoteRevokes: Map<number, { resolvers: (() => void)[]; at: number }> | undefined;
952
+ /**
953
+ * Bumped every time every relayed admission is fenced wholesale (`fenceAllRemoteAdmissions`, i.e. the
954
+ * owner worker is gone). `#acquireFromOwner` samples it before it asks the owner and re-checks after
955
+ * the reply: a grant minted by an owner that has since been declared gone must NOT be installed, or
956
+ * this thread would start writing under an admission the replacement owner knows nothing about while
957
+ * that owner, starting empty, grants the same key to somebody else. Carried forward on handoff so a
958
+ * transport swap mid-acquire cannot reset it and re-open the window.
959
+ */
960
+ #relayGeneration = 0;
961
+ #relayedAdmissions = 0;
962
+ /** Keys this node homes, and who holds each one. */
963
+ #grants = new Map<unknown, HomeGrant>();
964
+ /** Per-requester counts, so one peer cannot fill the home's table on its own. */
965
+ #grantsByRequester = new Map<string, number>();
966
+ /** Clean-handoff lineage outlives grants and is retained independently under its own cap. */
967
+ #dependencySets = new Map<unknown, LockDependencySet>();
968
+ #everDelegated = new DelegatedKeyFilter();
969
+ #freshnessGeneration: number | undefined;
970
+ /** False after any interval whose delegation history this coordinator could not have observed. */
971
+ #trustVirginKeys: boolean;
972
+ #counter = 0;
973
+ /**
974
+ * Closed coordinators must not keep admitting. `close()` sets this AND expires every delegation,
975
+ * because a handle already handed out checks only its own lease — clearing the table alone would
976
+ * let a successor coordinator (a re-registered transport) grant the same key with no lease time
977
+ * elapsed.
978
+ */
979
+ #closed = false;
980
+ // A latched warning would make a permanently misrouted deployment look like a quiet cluster.
981
+ #droppedOffOwner = 0;
982
+ #lastOffOwnerWarn = 0;
983
+
984
+ constructor(options: LockCoordinatorOptions) {
985
+ if (!isNodeName(options.nodeId) || NON_DISTINCTIVE_NODE_NAMES.has(options.nodeId))
986
+ throw new LockUnavailableError(
987
+ `Cluster record locks need a distinctive node name to derive a key's home from, but this node identifies as "${options.nodeId}". Set node.hostname to this node's name in system.hdb_nodes.`
988
+ );
989
+ this.database = options.database;
990
+ this.table = options.table;
991
+ liveCoordinators.add(this);
992
+ this.nodeId = options.nodeId;
993
+ this.transport = options.transport;
994
+ this.#writeControl = options.writeControl;
995
+ this.#keyIdOf = options.keyIdOf;
996
+ this.#nextTimestamp = options.nextTimestamp;
997
+ this.#monotonic = options.monotonic ?? (() => performance.now());
998
+ this.#skewMs = options.skewMs ?? LOCK_LEASE_SKEW_MS;
999
+ this.#autoTick = options.autoTick !== false;
1000
+ // Anchored at construction, which is the only instant core can prove nothing else was granting
1001
+ // under: process start is wrong (`performance.now()` is process-wide inside a worker too, so a
1002
+ // replacement coordinating thread would read an uptime far past it) and so is thread start (a
1003
+ // thread can take coordination ownership long after it booted). `adopt` and the retirement
1004
+ // record below waive or carry the horizon wherever a predecessor's authority is actually known.
1005
+ this.#quarantineWaived = options.grantableAfterMono !== undefined;
1006
+ this.#trustVirginKeys = this.#quarantineWaived;
1007
+ // Ownership observed here, not lazily on the first grant: a coordinator built while this thread
1008
+ // already coordinates has owned it since construction, and the construction horizon covers that.
1009
+ // Leaving it unset until a grant would date ownership from the grant and re-quarantine a reload.
1010
+ try {
1011
+ if (options.transport.ownsCoordination()) {
1012
+ this.#ownedSinceMono = this.#monotonic();
1013
+ // The incarnation it has been coordinating under, so the first grant does not read its own
1014
+ // construction as a takeover. Undefined when there is no map yet, which `#ownershipHorizon`
1015
+ // treats as a takeover — the conservative direction.
1016
+ this.#coordinatingIncarnation = options.transport.homeMap(options.database)?.homeIncarnation;
1017
+ }
1018
+ } catch {
1019
+ // A transport that cannot answer yet is not owning yet; `#ownershipHorizon` will observe it.
1020
+ }
1021
+ this.#grantableAfterMono = options.grantableAfterMono ?? this.#monotonic() + DELEGATION_LEASE_MS + this.#skewMs;
1022
+ options.adopt?.handOffTo(this);
1023
+ const retired = retiredCoordinators.get(this.#retirementKey());
1024
+ if (retired) {
1025
+ this.#counter = Math.max(this.#counter, retired.counter);
1026
+ this.#grantableAfterMono = Math.max(this.#grantableAfterMono, retired.grantableAfterMono);
1027
+ // A close without a successor discarded the key filter and retained dependency sets. An
1028
+ // explicit cold-start waiver on the replacement says nothing about that discarded history.
1029
+ this.#trustVirginKeys = false;
1030
+ }
1031
+ }
1032
+
1033
+ /**
1034
+ * Move this coordinator's live authority to the coordinator replacing it. Called from the
1035
+ * successor's constructor, before `close()`, so nothing is dropped in between. Both coordinators
1036
+ * run in the same thread for the same node, so the delegations and grants are still this node's —
1037
+ * only the transport object underneath them changed.
1038
+ */
1039
+ handOffTo(successor: LockCoordinator): void {
1040
+ if (this.#handedOff) return;
1041
+ this.#handedOff = true;
1042
+ // Kept so an acquisition still awaiting a home's reply can install it on whoever holds this
1043
+ // node's authority when the reply lands, rather than on a coordinator nothing consults.
1044
+ this.#successor = successor;
1045
+ for (const [keyId, delegation] of this.#delegations) successor.#delegations.set(keyId, delegation);
1046
+ for (const [keyId, pending] of this.#pendingDelegations) successor.#pendingDelegations.set(keyId, pending);
1047
+ for (const [keyId, request] of this.#pendingRequests) successor.#pendingRequests.set(keyId, request);
1048
+ for (const [keyId, grant] of this.#grants) successor.#grants.set(keyId, grant);
1049
+ for (const [requester, count] of this.#grantsByRequester) successor.#grantsByRequester.set(requester, count);
1050
+ // The admission index moves with the delegations it points into. Without it a handle admitted
1051
+ // on the predecessor could never be released through the successor — `release` addresses the
1052
+ // admission, so the entry would sit on the delegation forever and no recall could drain it.
1053
+ for (const [admissionId, delegation] of this.#admissions) successor.#admissions.set(admissionId, delegation);
1054
+ // Remote admissions (harper-pro#852) move with their handles: this thread still holds the native
1055
+ // key and the staged write for each, so the successor must be the one a later `release` or a
1056
+ // relayed `revoke` reaches. Their local ids stay valid (the successor's own `#nextAdmissionId` is
1057
+ // carried forward below), and the owner-id index and any not-yet-installed revokes move with them.
1058
+ if (this.#remoteAdmissions) {
1059
+ const into = (successor.#remoteAdmissions ??= new Map());
1060
+ for (const [localId, remote] of this.#remoteAdmissions) into.set(localId, remote);
1061
+ const ownerIndex = (successor.#remoteByOwnerId ??= new Map());
1062
+ for (const [ownerId, localId] of this.#remoteByOwnerId ?? []) ownerIndex.set(ownerId, localId);
1063
+ this.#remoteAdmissions = undefined;
1064
+ this.#remoteByOwnerId = undefined;
1065
+ }
1066
+ if (this.#pendingRemoteRevokes) {
1067
+ const into = (successor.#pendingRemoteRevokes ??= new Map());
1068
+ for (const [ownerId, entry] of this.#pendingRemoteRevokes) {
1069
+ const existing = into.get(ownerId);
1070
+ if (existing) existing.resolvers.push(...entry.resolvers);
1071
+ else into.set(ownerId, entry);
1072
+ }
1073
+ this.#pendingRemoteRevokes = undefined;
1074
+ }
1075
+ successor.#relayedAdmissions += this.#relayedAdmissions;
1076
+ // Never backwards: an acquire that sampled the predecessor must still see a bump the successor
1077
+ // (or the predecessor) already recorded, so a swap mid-acquire cannot re-open the orphan window.
1078
+ successor.#relayGeneration = Math.max(successor.#relayGeneration, this.#relayGeneration);
1079
+ // Neither counter may restart. A repeated token would compare equal to one the predecessor
1080
+ // already issued for a different delegation; a repeated admission id would address the wrong
1081
+ // admission in the map just carried over.
1082
+ successor.#counter = Math.max(successor.#counter, this.#counter);
1083
+ successor.#nextAdmissionId = Math.max(successor.#nextAdmissionId, this.#nextAdmissionId);
1084
+ // The predecessor's horizon EXACTLY, not the successor's freshly computed one. Adoption means the
1085
+ // successor now knows everything the predecessor knew, so it faces the same cold-start hazard and
1086
+ // no more: recomputing from its own construction would quarantine a node for a full delegation
1087
+ // lease on every transport reload, and clearing it would let a swap inside the window grant over
1088
+ // an unseen predecessor incarnation.
1089
+ successor.#grantableAfterMono = this.#grantableAfterMono;
1090
+ // And the ownership clock: a transport reload does not change which thread coordinates, so the
1091
+ // successor inherits how long this one has owned it rather than starting a fresh interval. The
1092
+ // waiver rides along with it and in the same direction: the successor reads `grantableAfterMono`
1093
+ // off the new transport and would otherwise re-waive a horizon this coordinator had already lost
1094
+ // to a takeover.
1095
+ //
1096
+ // Only what this coordinator actually OBSERVED may override that, though. `undefined` on either
1097
+ // of these means "never saw it" — a coordinator built on a non-owning thread, or before a map
1098
+ // was available — and the successor has just read the new transport, which is not less current.
1099
+ // Carrying the blank over it re-armed a quarantine on a node that had never stopped coordinating.
1100
+ if (this.#ownedSinceMono !== undefined) {
1101
+ successor.#ownedSinceMono = this.#ownedSinceMono;
1102
+ successor.#quarantineWaived = this.#quarantineWaived;
1103
+ }
1104
+ if (this.#coordinatingIncarnation !== undefined) successor.#coordinatingIncarnation = this.#coordinatingIncarnation;
1105
+ successor.#dependencySets = this.#dependencySets;
1106
+ successor.#everDelegated.copyFrom(this.#everDelegated);
1107
+ successor.#freshnessGeneration = this.#freshnessGeneration;
1108
+ successor.#trustVirginKeys = this.#trustVirginKeys;
1109
+ this.#dependencySets = new Map();
1110
+ this.#everDelegated.clear();
1111
+ this.#delegations.clear();
1112
+ this.#pendingDelegations.clear();
1113
+ this.#pendingRequests.clear();
1114
+ this.#grants.clear();
1115
+ this.#grantsByRequester.clear();
1116
+ this.#admissions.clear();
1117
+ if (
1118
+ successor.#delegations.size > 0 ||
1119
+ successor.#pendingDelegations.size > 0 ||
1120
+ successor.#grants.size > 0 ||
1121
+ (successor.#remoteAdmissions?.size ?? 0) > 0 ||
1122
+ (successor.#pendingRemoteRevokes?.size ?? 0) > 0
1123
+ )
1124
+ successor.#startTicking();
1125
+ }
1126
+
1127
+ /**
1128
+ * For `cluster_status`: delegations held, delegations issued, live admissions, admissions still
1129
+ * revocable, misrouted calls. `revocable` counts the ones an unlock left fenceable as well, so the
1130
+ * retention `#pruneAdmissions` bounds is visible rather than inferred from `admitted`.
1131
+ */
1132
+ get stats(): {
1133
+ delegations: number;
1134
+ granted: number;
1135
+ admitted: number;
1136
+ revocable: number;
1137
+ droppedOffOwner: number;
1138
+ relayedAdmissions: number;
1139
+ } {
1140
+ let admitted = 0;
1141
+ let revocable = 0;
1142
+ for (const delegation of this.#delegations.values()) {
1143
+ admitted += delegation.holding;
1144
+ revocable += delegation.admissions.size;
1145
+ }
1146
+ return {
1147
+ delegations: this.#delegations.size,
1148
+ granted: this.#grants.size,
1149
+ admitted,
1150
+ revocable,
1151
+ droppedOffOwner: this.#droppedOffOwner,
1152
+ // Admissions this thread has obtained from the owner worker (harper-pro#852), cumulative
1153
+ // rather than current: what it makes visible in `cluster_status` is that off-owner `lock()`s
1154
+ // are being served here at all.
1155
+ relayedAdmissions: this.#relayedAdmissions,
1156
+ };
1157
+ }
1158
+
1159
+ /**
1160
+ * Admit a critical section for a key whose native lock this thread already holds. Resolves with
1161
+ * the stamp and the monotonic reading the lease runs from; rejects 423 when no delegation could be
1162
+ * obtained in time, or 503 when the guarantee cannot be established at all.
1163
+ *
1164
+ * The amortization is the first branch: a live, un-recalled delegation with enough time left costs
1165
+ * zero cluster messages.
1166
+ */
1167
+ acquire(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
1168
+ return this.#acquire(key, leaseMs, waitMs);
1169
+ }
1170
+
1171
+ async #acquire(key: any, leaseMs: number, waitMs: number, observed?: LastCompletedReply): Promise<LockRound> {
1172
+ // `Table.lock()` captures a coordinator and only reaches here after the native key lock, which
1173
+ // can wait the caller's whole timeout — long enough for a transport swap to close what it
1174
+ // captured. Authority moved to the successor rather than away, so run there instead of
1175
+ // rejecting a caller that is already holding the key.
1176
+ const authority = this.#authority();
1177
+ if (authority !== this) return authority.#acquire(key, leaseMs, waitMs, observed);
1178
+ if (this.#closed) throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1179
+ if (!this.transport.ownsCoordination()) {
1180
+ // Off the coordinating thread. A transport that can relay obtains the admission from the
1181
+ // owner and installs the handle's revoker here (harper-pro#852); one that cannot fails
1182
+ // closed, the historical contract.
1183
+ // BOTH halves or neither. A transport that could acquire but not release would route locks and
1184
+ // then never forward an unlock, leaving every admission `holding` on the owner for its full
1185
+ // lease and stalling peer recalls — worse than the honest 503 below.
1186
+ if (this.transport.acquireOnOwner && this.transport.releaseOnOwner)
1187
+ return this.#acquireFromOwner(key, leaseMs, waitMs);
1188
+ throw new LockUnavailableError(
1189
+ 'Cluster record lock coordination is not owned by this worker thread; retry so the request reaches the coordinating thread'
1190
+ );
1191
+ }
1192
+ const keyId = this.#keyIdOf(key);
1193
+ const deadlineMono = this.#monotonic() + waitMs;
1194
+ let lastCompleted = observed;
1195
+
1196
+ acquisition: for (;;) {
1197
+ const homeMap = this.transport.homeMap(this.database);
1198
+ // No agreed map means no agreed ring, and a ring guessed from whoever looks reachable is
1199
+ // exactly the asymmetric-partition failure a single arbiter exists to remove.
1200
+ if (!homeMap || !this.#generationIsCurrent(homeMap.generation))
1201
+ throw new LockUnavailableError(
1202
+ `No agreed record lock home map for ${this.database}; cluster record locks are unavailable until one is established`
1203
+ );
1204
+ const delegation = this.#liveDelegation(keyId, leaseMs, homeMap.generation);
1205
+ if (delegation) return this.#admit(delegation, leaseMs);
1206
+
1207
+ const home = homeFor(this.#ringKey(keyId), homeMap.homes);
1208
+ if (!home)
1209
+ throw new LockUnavailableError(`The record lock home map for ${this.database} names no nodes to home a key on`);
1210
+
1211
+ // Anchored before the send: the home starts its own clock when it grants, so measuring the
1212
+ // delegation from the reply's arrival would hand a delayed reply more time than the home is
1213
+ // holding the key for.
1214
+ const requestedAtMono = this.#monotonic();
1215
+ const pendingRequest: PendingRequest = {};
1216
+ this.#pendingRequests.set(keyId, pendingRequest);
1217
+ let reply: DelegationReply;
1218
+ try {
1219
+ reply =
1220
+ home === this.nodeId
1221
+ ? this.#grantLocally(keyId, key, homeMap, leaseMs)
1222
+ : await this.#requestRemotely(home, keyId, key, homeMap, leaseMs, deadlineMono);
1223
+ } catch (error) {
1224
+ const requestAuthority = this.#authority();
1225
+ if (requestAuthority.#pendingRequests.get(keyId) === pendingRequest)
1226
+ requestAuthority.#pendingRequests.delete(keyId);
1227
+ throw error;
1228
+ }
1229
+
1230
+ // A transport swap can land while a request is in flight. The grant is authority for this
1231
+ // NODE, and the successor is this node now — installing it here would leave a delegation
1232
+ // nothing consults, admitting a caller that no recall can reach.
1233
+ let authority = this.#authority();
1234
+ if (authority.#pendingRequests.get(keyId) === pendingRequest) authority.#pendingRequests.delete(keyId);
1235
+ if (authority.#closed)
1236
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1237
+
1238
+ const replyDependencies =
1239
+ reply.dependencies === null ? null : normalizeDependencies(reply.dependencies, homeMap.homes);
1240
+ if (
1241
+ reply.granted &&
1242
+ isFencingToken(reply.token) &&
1243
+ isDuration(reply.leaseMs, MIN_LOCK_LEASE_MS, DELEGATION_LEASE_MS) &&
1244
+ replyDependencies !== undefined
1245
+ ) {
1246
+ const recalledBeforeReply =
1247
+ pendingRequest.recalledToken !== undefined && compareTokens(pendingRequest.recalledToken, reply.token) === 0;
1248
+ const existing = authority.#delegations.get(keyId);
1249
+ if (
1250
+ existing &&
1251
+ !existing.recalled &&
1252
+ existing.token[0] === reply.token[0] &&
1253
+ compareTokens(existing.token, reply.token) < 0
1254
+ ) {
1255
+ // This is a continuous renewal of authority this node already made fresh. No other
1256
+ // delegate could have held the key, so repeating the barrier — especially recovery —
1257
+ // buys no freshness. The home still returns its requirement for the case where the
1258
+ // requester lost local state and has no existing delegation to prove continuity.
1259
+ const renewed = authority.#installDelegation(
1260
+ keyId,
1261
+ key,
1262
+ reply.token,
1263
+ reply.leaseMs,
1264
+ requestedAtMono,
1265
+ existing.dependencies
1266
+ );
1267
+ if (renewed && recalledBeforeReply) {
1268
+ // The renewed delegation inherits every admission from the old token. Drain those
1269
+ // admissions through the ordinary recall path; surrendering directly would revoke
1270
+ // their handles and clear the home's grant while their critical sections still run.
1271
+ await authority.onDelegationRecall({ key, token: reply.token });
1272
+ if (authority.#closed)
1273
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1274
+ continue acquisition;
1275
+ }
1276
+ if (renewed && renewed.expiresMono - authority.#monotonic() >= leaseMs)
1277
+ return authority.#admit(renewed, leaseMs);
1278
+ continue acquisition;
1279
+ }
1280
+ const pending = createPendingDelegation(key, reply.token, recalledBeforeReply);
1281
+ authority.#pendingDelegations.set(keyId, pending);
1282
+ let dependencies: LockDependencySet;
1283
+ try {
1284
+ let requirement = replyDependencies;
1285
+ for (;;) {
1286
+ const barrierAuthority = authority;
1287
+ dependencies = await barrierAuthority.#establishFreshness(
1288
+ key,
1289
+ requirement,
1290
+ homeMap.homes,
1291
+ deadlineMono,
1292
+ pending
1293
+ );
1294
+ authority = this.#authority();
1295
+ if (authority === barrierAuthority) break;
1296
+ requirement = dependencies;
1297
+ }
1298
+ } catch (error) {
1299
+ authority = this.#authority();
1300
+ if (authority.#pendingDelegations.get(keyId) === pending) authority.#pendingDelegations.delete(keyId);
1301
+ await authority.#releaseUnclaimedGrant(key, reply.token);
1302
+ if (error === PENDING_DELEGATION_RECALLED) {
1303
+ if (authority.#closed)
1304
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1305
+ continue acquisition;
1306
+ }
1307
+ throw new LockUnavailableError(
1308
+ `Could not establish successor freshness for ${this.database}.${this.table}: ${(error as Error)?.message ?? error}`
1309
+ );
1310
+ }
1311
+ const currentPending = authority.#pendingDelegations.get(keyId);
1312
+ if (currentPending === pending) authority.#pendingDelegations.delete(keyId);
1313
+ if (currentPending !== pending || pending.recalled) {
1314
+ await authority.#releaseUnclaimedGrant(key, reply.token);
1315
+ if (authority.#closed)
1316
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1317
+ continue;
1318
+ }
1319
+ // The generation can advance across the await. A grant minted under a superseded one is
1320
+ // authority for a ring that no longer exists: the key may be homed elsewhere now, and that
1321
+ // home can already have granted it to another node. `#liveDelegation` rejects a stale
1322
+ // token on the NEXT pass, which is too late — this pass would have admitted on it first.
1323
+ if (reply.token[0] !== authority.transport.homeMap(this.database)?.generation) {
1324
+ // Hand it back rather than let the old home hold a key nobody is using for a full lease.
1325
+ authority.#releaseUnclaimedGrant(key, reply.token);
1326
+ } else {
1327
+ const installed = authority.#installDelegation(
1328
+ keyId,
1329
+ key,
1330
+ reply.token,
1331
+ reply.leaseMs,
1332
+ requestedAtMono,
1333
+ dependencies
1334
+ );
1335
+ // A reply that outlived its own delegation grants nothing; fall through and ask again
1336
+ // rather than admitting on authority the home has already expired.
1337
+ if (installed && !installed.recalled && installed.expiresMono - authority.#monotonic() >= leaseMs)
1338
+ return authority.#admit(installed, leaseMs);
1339
+ // A long barrier can consume enough of the delegation that the requested lease no longer
1340
+ // fits. Surrender the unused installed delegation before retrying; otherwise the home renews
1341
+ // it in place on every pass and contenders wait for its full deadline.
1342
+ if (installed && compareTokens(installed.token, reply.token) === 0 && installed.holding === 0)
1343
+ await authority.#surrender(keyId, installed);
1344
+ else if (!installed) await authority.#releaseUnclaimedGrant(key, reply.token);
1345
+ }
1346
+ } else if (reply.granted) {
1347
+ if (isFencingToken(reply.token)) authority.#releaseUnclaimedGrant(key, reply.token);
1348
+ throw new LockUnavailableError(
1349
+ `The home node for this key on ${this.database}.${this.table} returned a delegation with no usable token or freshness requirement`
1350
+ );
1351
+ }
1352
+ if (reply.reason === 'capacity')
1353
+ throw new LockUnavailableError(`Too many record lock delegations in flight on ${this.database}`);
1354
+ if (reply.reason === 'quarantine')
1355
+ throw new LockUnavailableError(
1356
+ `The home node for this key on ${this.database}.${this.table} restarted and cannot grant until the delegations its previous incarnation issued have expired`
1357
+ );
1358
+ // Neither of these can be waited out inside a `lock()` timeout, and retrying them spends the
1359
+ // caller's whole budget holding the native key only to answer 423 for a key nobody holds.
1360
+ if (reply.reason === 'generation')
1361
+ throw new LockUnavailableError(
1362
+ `The home node for this key on ${this.database}.${this.table} holds record lock home map generation ${reply.generation ?? 'unknown'} and this node holds ${homeMap.generation}; cluster record locks are unavailable until they agree`
1363
+ );
1364
+ if (reply.reason === 'unknown-node')
1365
+ throw new LockUnavailableError(
1366
+ `This node is not named in the record lock home map for ${this.database}, so it can neither home a key nor take a cluster lock on one`
1367
+ );
1368
+ if (reply.reason === 'not-home')
1369
+ // The home disagrees about the ring. Re-reading the map on the next pass is the fix, and it
1370
+ // converges if our copy is the stale one — so unlike the denials above this one is worth
1371
+ // retrying. If it is NOT stale (two maps under one generation number) it never converges,
1372
+ // which the terminal answer below is what handles.
1373
+ warnOnce('record lock home disagreed about the ring', { database: this.database, table: this.table });
1374
+
1375
+ if (reply.reason !== 'timeout') lastCompleted = { reply, home, generation: homeMap.generation };
1376
+ const remaining = deadlineMono - this.#monotonic();
1377
+ const retryAfterMs = reply.retryAfterMs ?? 25;
1378
+ // A home this node IS costs nothing to ask again — `#grantLocally` is synchronous, so a release
1379
+ // landing in the backoff is still grantable at the deadline. A remote home is not: its request
1380
+ // would go out with the leftover budget and could only return this node's own `timeout`.
1381
+ const exhausted = home === this.nodeId ? remaining <= 0 : remaining <= retryAfterMs;
1382
+ if (!exhausted) await delay(Math.min(retryAfterMs, remaining)).promise;
1383
+ if (this.#closed) {
1384
+ // Same swap, landing in the backoff instead. The successor inherits both the remaining wait
1385
+ // and what this one saw, since a successor exhausted on arrival has nothing of its own.
1386
+ const successor = this.#authority();
1387
+ if (successor === this)
1388
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1389
+ return successor.#acquire(key, leaseMs, Math.max(0, deadlineMono - this.#monotonic()), lastCompleted);
1390
+ }
1391
+ if (!exhausted) continue acquisition;
1392
+ // Only an observation made under the generation that is current NOW still describes the key —
1393
+ // this pass's reply included, since a generation can be activated while the probe that ended
1394
+ // the wait is still in flight. Hence a fresh read rather than the copy this pass started from.
1395
+ const currentGeneration = this.transport.homeMap(this.database)?.generation;
1396
+ const carried =
1397
+ lastCompleted?.home === home && lastCompleted.generation === currentGeneration
1398
+ ? lastCompleted.reply
1399
+ : undefined;
1400
+ const terminal = carried ?? (homeMap.generation === currentGeneration ? reply : undefined);
1401
+ // Only `contended` may end as 423: it is the one answer that says another node holds the key,
1402
+ // and reporting contention for anything else sends the caller to retry a condition no wait
1403
+ // outlasts. A wait can see `contended` and still end 503 — the rule is what the home said LAST.
1404
+ if (terminal?.reason === 'contended') throw new ClientError('Record is locked and was not released in time', 423);
1405
+ throw new LockUnavailableError(
1406
+ `Could not establish a cluster record lock on ${this.database}.${this.table} within the wait: ${describeExhaustedWait(terminal, currentGeneration !== undefined)}`
1407
+ );
1408
+ }
1409
+ }
1410
+
1411
+ /**
1412
+ * Obtain an admission from the owner worker for a `lock()` served on a non-owner thread
1413
+ * (harper-pro#852), recorded under a LOCAL id so `registerAdmission`/`release` can never collide it
1414
+ * with a live local admission. `acquireOnOwner` is bounded by the transport, inside the caller's
1415
+ * `waitMs`; the race here is core's own backstop against a transport that never answers, so `lock()`
1416
+ * cannot hang far past the wait it was given.
1417
+ */
1418
+ async #acquireFromOwner(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
1419
+ // The transport bounds this wait (harper-pro's acquire has its own `waitMs`-scaled timeout); the
1420
+ // core-side race is a last-resort backstop set strictly beyond that bound — the transport is asked
1421
+ // for `waitMs - hop` and core's deadline is `waitMs + REMOTE_ACQUIRE_BACKSTOP_MS` — so it fires only
1422
+ // if the transport never returns, never ahead of the transport's own timeout. Without it a wedged
1423
+ // owner would hang `lock()` past its `waitMs`.
1424
+ // Sampled BEFORE the request goes out, on the authority that will install the result.
1425
+ const startGeneration = this.#authority().#relayGeneration;
1426
+ // The hop allowance comes OUT of the caller's budget, never on top of it: `lock()` holds the native
1427
+ // key for this whole wait, so overshooting `waitMs` blocks every other worker on the key for the
1428
+ // overshoot. The transport is asked for the reduced wait and answers within the caller's budget;
1429
+ // core's deadline is the caller's `waitMs` plus a small net that fires only if the transport never
1430
+ // answers at all.
1431
+ const hop = Math.min(REMOTE_ACQUIRE_HOP_MS, Math.floor(waitMs / 2));
1432
+ // A transport that throws SYNCHRONOUSLY (a sibling port already gone) must still reach the
1433
+ // normalization below: escaping raw gives the caller a 500 where it needs the retryable 503.
1434
+ // `#beginRecall` wraps its transport call for the same reason.
1435
+ const acquire = Promise.resolve().then(() =>
1436
+ this.transport.acquireOnOwner!(this.database, this.table, key, leaseMs, waitMs - hop)
1437
+ );
1438
+ let backstopWon = false;
1439
+ // If the backstop wins the race, the owner may still grant afterward: release that grant back so it
1440
+ // does not sit `holding` on the owner for its whole lease. (The transport releases a late reply too;
1441
+ // this closes the case where core's backstop fired first.)
1442
+ acquire.then(
1443
+ (round) => {
1444
+ if (backstopWon) this.#authority().#releaseOnOwnerSafely(key, round.admissionId);
1445
+ },
1446
+ () => {}
1447
+ );
1448
+ const round = await withDeadline(
1449
+ acquire,
1450
+ waitMs + REMOTE_ACQUIRE_BACKSTOP_MS,
1451
+ 'the coordinating worker did not answer'
1452
+ ).catch((error) => {
1453
+ backstopWon = true;
1454
+ throw error instanceof LockUnavailableError
1455
+ ? error
1456
+ : new LockUnavailableError(
1457
+ `Could not obtain a cluster record lock on ${this.database}.${this.table} from the coordinating worker: ${(error as Error)?.message ?? error}`
1458
+ );
1459
+ });
1460
+ const authority = this.#authority();
1461
+ if (authority.#closed) {
1462
+ // The coordinator closed while the owner was granting. The owner still holds this admission;
1463
+ // hand it straight back rather than leaving it outstanding for its whole lease.
1464
+ authority.#releaseOnOwnerSafely(key, round.admissionId);
1465
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1466
+ }
1467
+ if (authority.#relayGeneration !== startGeneration) {
1468
+ // Every relayed admission was fenced while this grant was in flight: the owner that minted it
1469
+ // has been declared gone, and the replacement starts with no record of it. Installing it now
1470
+ // would let this thread write under an admission the new owner can neither see nor recall,
1471
+ // while that owner grants the same key to another worker — two writers. Fail closed and hand
1472
+ // the grant back; the caller retries against the new owner.
1473
+ authority.#releaseOnOwnerSafely(key, round.admissionId);
1474
+ throw new LockUnavailableError(
1475
+ 'The record lock coordinating worker changed while this lock was being granted; retry against the new owner'
1476
+ );
1477
+ }
1478
+ // A live entry already keyed to this owner id means the owner reused an id its counter had already
1479
+ // issued — only possible if a replacement owner restarted its sequence. The old entry is from that
1480
+ // prior owner incarnation and its delegation is gone: fence it fail-closed before the new one takes
1481
+ // the id, so a later revoke for this id can never address the stale handle.
1482
+ const collidingLocalId = authority.#remoteByOwnerId?.get(round.admissionId);
1483
+ if (collidingLocalId !== undefined) {
1484
+ const stale = authority.#remoteAdmissions?.get(collidingLocalId);
1485
+ if (stale) {
1486
+ fireRevokeAndForget(stale.revoke);
1487
+ authority.#dropRemoteAdmission(collidingLocalId, stale);
1488
+ }
1489
+ }
1490
+ const localId = authority.#nextAdmissionId++;
1491
+ const entry: RemoteAdmission = {
1492
+ ownerAdmissionId: round.admissionId,
1493
+ key,
1494
+ revoke: noRevoke,
1495
+ expiresMono: round.mintedMono + leaseMs,
1496
+ revoked: false,
1497
+ fenceWaiters: [],
1498
+ };
1499
+ (authority.#remoteAdmissions ??= new Map()).set(localId, entry);
1500
+ (authority.#remoteByOwnerId ??= new Map()).set(round.admissionId, localId);
1501
+ // A revoke that raced ahead of this install (the owner registered its revoker before it posted
1502
+ // the grant reply, so a recall in that window fired first): apply it now so the ack the owner is
1503
+ // waiting on cannot resolve before the handle is fenced.
1504
+ const pending = authority.#pendingRemoteRevokes?.get(round.admissionId);
1505
+ if (pending) {
1506
+ authority.#pendingRemoteRevokes!.delete(round.admissionId);
1507
+ entry.revoked = true;
1508
+ entry.fenceWaiters.push(...pending.resolvers);
1509
+ }
1510
+ authority.#relayedAdmissions++;
1511
+ // Tick so an entry whose caller never releases (its lease elapses instead) is still pruned.
1512
+ authority.#startTicking();
1513
+ // The LOCAL id is what the handle registers and releases against; the round the owner minted
1514
+ // still carries its own id, which only the owner-facing release/revoke messages use.
1515
+ return { tsR: round.tsR, mintedMono: round.mintedMono, admissionId: localId };
1516
+ }
1517
+
1518
+ /**
1519
+ * Fence a remote admission's handle on this thread, driven by a recall or surrender on the owner
1520
+ * (harper-pro#852), addressed by the OWNER's admission id. Resolves once the handle is provably
1521
+ * fenced — its `revokeLease` has run — so the owner's ack cannot claim a fence the handle has not
1522
+ * taken; the owner waits on this before writing the release. A revoke that arrives before the handle
1523
+ * is installed, or before `registerAdmission` supplies the real revoker, resolves only when the fence
1524
+ * finally lands. A revoker that throws, or whose promise rejects, fails this promise rather than
1525
+ * resolving it, so the owner falls back to its own lease bound rather than being told the fence
1526
+ * succeeded.
1527
+ */
1528
+ revokeRemoteAdmission(ownerAdmissionId: number): Promise<void> {
1529
+ const localId = this.#remoteByOwnerId?.get(ownerAdmissionId);
1530
+ if (localId === undefined) {
1531
+ // The entry is not installed yet (a revoke that raced ahead of the grant reply): latch the ack
1532
+ // resolver so `#acquireFromOwner` can carry it, and never fence-then-resolve a handle that does
1533
+ // not exist. If the acquire never lands (a revoke for an admission already dropped), `tick`
1534
+ // resolves and sweeps it after a lease so the resolver cannot leak.
1535
+ return new Promise<void>((resolve) => {
1536
+ const map = (this.#pendingRemoteRevokes ??= new Map());
1537
+ const pending = map.get(ownerAdmissionId);
1538
+ if (pending) pending.resolvers.push(resolve);
1539
+ else map.set(ownerAdmissionId, { resolvers: [resolve], at: this.#monotonic() });
1540
+ this.#startTicking();
1541
+ });
1542
+ }
1543
+ const remote = this.#remoteAdmissions?.get(localId);
1544
+ if (!remote) return Promise.resolve();
1545
+ remote.revoked = true;
1546
+ if (remote.revoke === noRevoke) {
1547
+ // The handle has not registered its revoker yet; resolve when `registerAdmission` fences it.
1548
+ return new Promise<void>((resolve) => remote.fenceWaiters.push(resolve));
1549
+ }
1550
+ return Promise.resolve(this.#fenceRemoteAdmission(localId, remote));
1551
+ }
1552
+
1553
+ /**
1554
+ * Fire a remote admission's revoker and report when the handle is PROVABLY fenced. `registerAdmission`
1555
+ * accepts an async revoker, so an outcome that is promise-like is not a fence until it fulfils: the
1556
+ * entry is dropped — and any ack waiting on it settled — only then. A rejection is not a fence at all,
1557
+ * so it propagates and the entry is LEFT for the lease sweep: a retry can fire the revoker again, and
1558
+ * the owner either sees the failure or waits its own lease bound out rather than writing the release
1559
+ * and admitting a successor over a live writer. Unlike a plain release, which retains the entry
1560
+ * because a staged write is still committable until the lease.
1561
+ */
1562
+ #fenceRemoteAdmission(localId: number, remote: RemoteAdmission): void | Promise<void> {
1563
+ let fenced: void | Promise<void>;
1564
+ try {
1565
+ fenced = remote.revoke();
1566
+ } catch (error) {
1567
+ // A synchronous throw is not a fence either, and it must not escape past the promise this
1568
+ // function's callers hand to the owner: surface it as a rejection instead.
1569
+ warnOnce('a relayed record lock fence failed; the handle lease still bounds it', error);
1570
+ return Promise.reject(error);
1571
+ }
1572
+ if (!isPromiseLike(fenced)) {
1573
+ this.#dropRemoteAdmission(localId, remote);
1574
+ return;
1575
+ }
1576
+ return fenced.then(
1577
+ () => {
1578
+ this.#dropRemoteAdmission(localId, remote);
1579
+ },
1580
+ (error) => {
1581
+ warnOnce('a relayed record lock fence failed; the handle lease still bounds it', error);
1582
+ throw error;
1583
+ }
1584
+ );
1585
+ }
1586
+
1587
+ /**
1588
+ * Fence every relayed handle this thread holds, fail-closed (harper-pro#852). Called when the owner
1589
+ * worker that granted them is gone (its thread exited, so its delegation table died with it): the
1590
+ * handles are no longer backed by any delegation, so nothing here may keep committing. There is no
1591
+ * owner left to tell, so this only revokes locally and drops the entries; a still-pending ack is
1592
+ * resolved, since the handle can no longer commit.
1593
+ */
1594
+ fenceAllRemoteAdmissions(): void {
1595
+ // Before anything is torn down, so an acquire already in flight to the departed owner fails its
1596
+ // post-reply generation check rather than installing an orphaned admission.
1597
+ this.#relayGeneration++;
1598
+ if (this.#remoteAdmissions) {
1599
+ for (const remote of this.#remoteAdmissions.values()) {
1600
+ fireRevokeAndForget(remote.revoke);
1601
+ this.#settleFenceWaiters(remote);
1602
+ }
1603
+ this.#remoteAdmissions = undefined;
1604
+ this.#remoteByOwnerId = undefined;
1605
+ }
1606
+ if (this.#pendingRemoteRevokes) {
1607
+ for (const entry of this.#pendingRemoteRevokes.values()) for (const resolve of entry.resolvers) resolve();
1608
+ this.#pendingRemoteRevokes = undefined;
1609
+ }
1610
+ }
1611
+
1612
+ /**
1613
+ * Forward a release to the owner, naming the admission by the OWNER's id. Core deliberately carries no
1614
+ * owner epoch of its own: a release for an id a replacement owner has since reused is rejected by the
1615
+ * transport, which stamps every release with the owner session captured when the admission was minted
1616
+ * and drops its cached sessions when the coordinating thread changes. An owner ignores anything not
1617
+ * stamped with its own session, so a stale release cannot address a live admission that reuses the id.
1618
+ */
1619
+ #releaseOnOwnerSafely(key: any, ownerAdmissionId: number): void {
1620
+ // The common relayed unlock is a synchronous post to the owner thread, so it must not cost a
1621
+ // promise chain and two microtasks per unlock: call it directly and only attach a rejection
1622
+ // handler when the transport actually returned a promise.
1623
+ try {
1624
+ const outcome = this.transport.releaseOnOwner?.(this.database, this.table, key, ownerAdmissionId);
1625
+ if (isPromiseLike(outcome))
1626
+ outcome.catch((error) => warnOnce('failed to forward a relayed record lock release to the owner', error));
1627
+ } catch (error) {
1628
+ warnOnce('failed to forward a relayed record lock release to the owner', error);
1629
+ }
1630
+ }
1631
+
1632
+ /** Resolve every ack waiting on a remote admission's fence, then forget them. */
1633
+ #settleFenceWaiters(remote: RemoteAdmission): void {
1634
+ if (remote.fenceWaiters.length === 0) return;
1635
+ const waiters = remote.fenceWaiters;
1636
+ remote.fenceWaiters = [];
1637
+ for (const resolve of waiters) resolve();
1638
+ }
1639
+
1640
+ /**
1641
+ * Drop remote admissions whose handle lease elapsed a grace ago, at most `MAX_EXPIRIES_PER_TICK` per
1642
+ * tick like the delegation and grant sweeps — a burst of tens of thousands of same-lease admissions
1643
+ * expiring together drains over several ticks rather than deleting them all (and firing a release
1644
+ * message apiece) in one 100 ms turn. The scan skips still-live entries so different lease lengths are
1645
+ * handled. `REMOTE_PRUNE_GRACE_MS` past the lease is what lets the handle's OWN lease timer fire first
1646
+ * and forward its release (`release`) rather than racing this sweep.
1647
+ *
1648
+ * Entries EXAMINED are bounded too, not just deletions: a thread holding many live relayed admissions
1649
+ * would otherwise walk every one of them on every 100 ms tick, purely to find nothing. What is left
1650
+ * unexamined is reached on a later tick, which is safe because this sweep only reclaims memory — an
1651
+ * admission's handle is fenced by its own lease, never by this loop running promptly.
1652
+ *
1653
+ * The scan restarts at the map head each tick rather than carrying a cursor, so above the examine cap
1654
+ * a long-lived prefix would delay collecting expired entries behind it. Accepted deliberately: the cap
1655
+ * is far above any realistic count of concurrent off-owner locks on one thread, and a cursor would
1656
+ * have to walk the prefix anyway to find its resume point, spending the work it meant to save. It
1657
+ * costs memory held longer in a case that should not arise, never correctness.
1658
+ */
1659
+ #pruneRemoteAdmissions(now: number): void {
1660
+ const remotes = this.#remoteAdmissions;
1661
+ if (!remotes) return;
1662
+ const horizon = now - REMOTE_PRUNE_GRACE_MS;
1663
+ let budget = MAX_EXPIRIES_PER_TICK;
1664
+ let examined = MAX_EXAMINED_PER_TICK;
1665
+ for (const [localId, remote] of remotes) {
1666
+ if (budget <= 0 || examined <= 0) break;
1667
+ examined--;
1668
+ if (remote.expiresMono > horizon) continue;
1669
+ budget--;
1670
+ this.#dropRemoteAdmission(localId, remote, true);
1671
+ }
1672
+ if (remotes.size === 0) {
1673
+ this.#remoteAdmissions = undefined;
1674
+ this.#remoteByOwnerId = undefined;
1675
+ }
1676
+ }
1677
+
1678
+ /**
1679
+ * Drop a remote admission entry. `forwardRelease` tells the owner to give the delegation up now —
1680
+ * set when pruning at lease, since an event-loop stall can let this sweep run before the handle's
1681
+ * own lease timer forwards the release, and the owner would otherwise hold `holding` until its own
1682
+ * delegation lease. The forward is idempotent on the owner, so a race with the handle's own release
1683
+ * is harmless. Not set for a fence-drop: the owner is already surrendering that delegation.
1684
+ */
1685
+ #dropRemoteAdmission(localId: number, remote: RemoteAdmission, forwardRelease = false): void {
1686
+ // Optional: an asynchronous fence can settle after `fenceAllRemoteAdmissions`, `close` or a
1687
+ // handoff has already cleared the map, and a TypeError here would reject an ack whose handle is
1688
+ // in fact fenced, making the owner wait out the lease for nothing.
1689
+ this.#remoteAdmissions?.delete(localId);
1690
+ this.#remoteByOwnerId?.delete(remote.ownerAdmissionId);
1691
+ // The handle's own lease has elapsed, so it fences itself; a still-pending ack may resolve.
1692
+ this.#settleFenceWaiters(remote);
1693
+ if (forwardRelease) this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
1694
+ }
1695
+
1696
+ /**
1697
+ * Resolve and drop a latched revoke whose admission never installed — a revoke for an admission this
1698
+ * thread had already dropped (fenced or pruned), so the handle is gone and the ack is vacuously
1699
+ * satisfied. Bounded by a lease: past that, no acquire can still be in flight for the id.
1700
+ */
1701
+ #prunePendingRemoteRevokes(now: number): void {
1702
+ const pending = this.#pendingRemoteRevokes;
1703
+ if (!pending) return;
1704
+ // Bounded like the admission sweep above, and for the same reason: what this tick does not reach,
1705
+ // a later one does. Each waiting ack is independently bounded by the owner's own lease wait.
1706
+ let examined = MAX_EXAMINED_PER_TICK;
1707
+ for (const [ownerId, entry] of pending) {
1708
+ if (examined <= 0) break;
1709
+ examined--;
1710
+ if (now - entry.at < MAX_LOCK_LEASE_MS) continue;
1711
+ pending.delete(ownerId);
1712
+ for (const resolve of entry.resolvers) resolve();
1713
+ }
1714
+ if (pending.size === 0) this.#pendingRemoteRevokes = undefined;
1715
+ }
1716
+
1717
+ /**
1718
+ * End this node's admission for a key. The delegation is deliberately KEPT: that is the
1719
+ * amortization, and the next `lock()` on this node costs nothing. Returns the durable release
1720
+ * write only when the delegation is actually being given up.
1721
+ */
1722
+ release(key: any, admissionId: number): Promise<void> | void {
1723
+ // A remote admission (harper-pro#852) — the common path once threads.count > 1 — tells the owner
1724
+ // this caller unlocked (its own id, not this thread's local one) but keeps the entry, revoker
1725
+ // included, until the handle's lease runs out: §6 revokes CAPABILITY, not admission, so a write
1726
+ // staged before `unlock()` stays fenceable. Checked before `#keyIdOf` so a relayed unlock does not
1727
+ // pay that key encoding for nothing.
1728
+ const remote = this.#remoteAdmissions?.get(admissionId);
1729
+ if (remote) {
1730
+ this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
1731
+ return undefined;
1732
+ }
1733
+ const keyId = this.#keyIdOf(key);
1734
+ // Addressed by admission, not by key: after a renewal or a replacement the delegation at this
1735
+ // key may not be the one that admitted this handle, and releasing by key alone would either
1736
+ // surrender a successor while its own callers were still inside, or leave this admission on a
1737
+ // delegation nobody can ever drain.
1738
+ const delegation = this.#admissions.get(admissionId);
1739
+ if (!delegation) return undefined;
1740
+ const admission = delegation.admissions.get(admissionId);
1741
+ if (!admission?.holding) return undefined;
1742
+ // The entry stays. Unlocking ends this caller's claim on the DRAIN; the write it staged can
1743
+ // still commit, so the revoker has to remain reachable until the handle's own lease runs out.
1744
+ admission.holding = false;
1745
+ delegation.holding--;
1746
+ if (delegation.holding > 0) return undefined;
1747
+ if (delegation.drained) {
1748
+ // Waking the recall is enough: it surrenders once, on its own path. Surrendering here too
1749
+ // would write a second release entry for the same handoff.
1750
+ const waiters = delegation.drained;
1751
+ delegation.drained = undefined;
1752
+ for (const resolve of waiters) resolve();
1753
+ return undefined;
1754
+ }
1755
+ // Recalled with nobody waiting on the drain — the recall already returned, so this is the path
1756
+ // that gives the delegation up. An un-recalled delegation is deliberately retained.
1757
+ if (delegation.recalled && this.#delegations.get(keyId) === delegation) return this.#surrender(keyId, delegation);
1758
+ return undefined;
1759
+ }
1760
+
1761
+ /**
1762
+ * Apply a control entry from a peer (or a replayed one). Idempotent by `(requester, tsR)`: a
1763
+ * release that arrives twice, or is replayed from the log long after its producer is gone, must be
1764
+ * harmless.
1765
+ *
1766
+ * `author` is the node the entry was actually written by, taken from the audit header rather than
1767
+ * the payload — without it a peer could write a release naming any other node and clear a
1768
+ * delegation it does not hold.
1769
+ */
1770
+ applyEntry(entry: LockControlEntry, author: string, position?: number): void {
1771
+ // The only boundary peer input crosses into this state machine. A throw here would reach the
1772
+ // replicated apply loop and drop the whole enclosing transaction, so one malformed entry could
1773
+ // stall replication for the database.
1774
+ try {
1775
+ this.#applyEntry(entry, author, position);
1776
+ } catch (error) {
1777
+ warnOnce('failed to apply a record lock control entry', error);
1778
+ }
1779
+ }
1780
+
1781
+ #applyEntry(entry: LockControlEntry, author: string, position?: number): void {
1782
+ if (this.#closed) return;
1783
+ if (entry.type !== 'lockRelease' || !isNodeName(author)) return;
1784
+ if (entry.requester !== author) return;
1785
+ if (!this.transport.ownsCoordination()) {
1786
+ this.#droppedOffOwner++;
1787
+ const now = this.#monotonic();
1788
+ if (now - this.#lastOffOwnerWarn > WARN_INTERVAL_MS) {
1789
+ this.#lastOffOwnerWarn = now;
1790
+ harperLogger.warn?.('record lock control entries are reaching a non-coordinating thread', {
1791
+ database: this.database,
1792
+ table: this.table,
1793
+ dropped: this.#droppedOffOwner,
1794
+ });
1795
+ }
1796
+ return;
1797
+ }
1798
+ if (!isFencingToken(entry.token)) return;
1799
+ const keyId = this.#keyIdOf(entry.key);
1800
+ const grant = this.#grants.get(keyId);
1801
+ // Only the delegate named in the live grant can clear it, and only for the exact token it was
1802
+ // issued — generation and incarnation included, since counters restart. A delayed release from a
1803
+ // previous delegation must not clear its successor's.
1804
+ if (!grant || grant.delegate !== author || compareTokens(grant.token, entry.token) !== 0) return;
1805
+ if (entry.dependencies !== null) {
1806
+ const homeMap = this.transport.homeMap(this.database);
1807
+ const matchingHomes = homeMap?.generation === entry.token[0] ? homeMap.homes : undefined;
1808
+ const inherited = matchingHomes ? normalizeDependencies(entry.dependencies, matchingHomes) : undefined;
1809
+ if (
1810
+ inherited &&
1811
+ matchingHomes.includes(author) &&
1812
+ typeof position === 'number' &&
1813
+ Number.isFinite(position) &&
1814
+ position >= 0
1815
+ ) {
1816
+ const merged = new Map<string, number>(this.#dependencySets.get(keyId));
1817
+ for (const [origin, dependencyPosition] of inherited)
1818
+ merged.set(origin, Math.max(merged.get(origin) ?? -Infinity, dependencyPosition));
1819
+ merged.set(author, Math.max(merged.get(author) ?? -Infinity, position));
1820
+ if (merged.size <= MAX_LOCK_DEPENDENCIES)
1821
+ this.#rememberDependencies(
1822
+ keyId,
1823
+ [...merged].sort(([a], [b]) => a.localeCompare(b))
1824
+ );
1825
+ else this.#dependencySets.delete(keyId);
1826
+ } else {
1827
+ this.#dependencySets.delete(keyId);
1828
+ }
1829
+ } else if (grant.renewed) this.#dependencySets.delete(keyId);
1830
+ this.#clearGrant(keyId, grant);
1831
+ }
1832
+
1833
+ /**
1834
+ * Inbound delegation request from a peer, for a key this node homes. The home is the single
1835
+ * arbiter, so this is the whole of the exclusion argument: one live grant per key, and a successor
1836
+ * only after the predecessor is recalled-and-drained or provably expired here.
1837
+ */
1838
+ async onDelegationRequest(request: DelegationRequest): Promise<DelegationReply> {
1839
+ if (this.#closed || !this.transport.ownsCoordination()) return { granted: false, reason: 'not-home' };
1840
+ if (!isNodeName(request.requester) || !isEncodableKey(request.key)) return { granted: false, reason: 'not-home' };
1841
+ if (!isDuration(request.leaseMs, MIN_LOCK_LEASE_MS, MAX_LOCK_LEASE_MS))
1842
+ return { granted: false, reason: 'not-home' };
1843
+ const homeMap = this.transport.homeMap(this.database);
1844
+ if (!homeMap || !this.#generationIsCurrent(homeMap.generation)) return { granted: false, reason: 'generation' };
1845
+ if (homeMap.generation !== request.generation)
1846
+ return { granted: false, reason: 'generation', generation: homeMap.generation };
1847
+ // Membership before state: a node the current map does not name has no claim on a key, and an
1848
+ // authenticated replication identity outlives membership. Without this a decommissioned node
1849
+ // takes delegations against live homes and recalls the legitimate delegate to get them. This is
1850
+ // why `homes` has to name every node that takes a cluster lock, not only the arbiters — see
1851
+ // `LockHomeMap.homes`.
1852
+ if (!homeMap.homes.includes(request.requester)) return { granted: false, reason: 'unknown-node' };
1853
+ // Both sides must agree we are the home, or two arbiters could issue for one key.
1854
+ const keyId = this.#keyIdOf(request.key);
1855
+ if (homeFor(this.#ringKey(keyId), homeMap.homes) !== this.nodeId) return { granted: false, reason: 'not-home' };
1856
+ return this.#grant(keyId, request.key, homeMap, request.leaseMs, request.requester);
1857
+ }
1858
+
1859
+ /**
1860
+ * Inbound recall from a key's home. Revokes capability rather than closing the door: no new
1861
+ * admission may start, live ones are drained, and the release is written only once the delegation
1862
+ * can no longer admit or commit. A recall for a token we no longer hold is a no-op, not an error.
1863
+ */
1864
+ async onDelegationRecall(recall: DelegationRecall): Promise<void> {
1865
+ // A close that was not a handoff expired and revoked every delegation first, so nothing here can
1866
+ // admit on any token and resolving is honest. A handoff does not reach this: the table's getter
1867
+ // answers the successor, which adopted them.
1868
+ if (this.#closed) return;
1869
+ // Ownership-gated like `onDelegationRequest`, and for the stronger reason: `acquire` refuses off
1870
+ // the owner thread, so a delegation only ever lives on the coordinating one. A recall routed to
1871
+ // any other thread finds no delegation and would resolve — which the home reads as a drained
1872
+ // delegate — while the real delegate keeps admitting. Resolving has to mean nothing on THIS NODE
1873
+ // can admit on that token, and only the owner can say so.
1874
+ if (!this.transport.ownsCoordination())
1875
+ throw new Error('Cluster record lock coordination is not owned by this worker thread');
1876
+ const keyId = this.#keyIdOf(recall.key);
1877
+ const pending = this.#pendingDelegations.get(keyId);
1878
+ if (pending && compareTokens(pending.token, recall.token) === 0) {
1879
+ pending.markRecalled();
1880
+ return;
1881
+ }
1882
+ const delegation = this.#delegations.get(keyId);
1883
+ if (!delegation || compareTokens(delegation.token, recall.token) !== 0) {
1884
+ // A recall can beat the grant reply. Remember its token so that reply cannot install authority
1885
+ // the home already believes drained.
1886
+ const request = this.#pendingRequests.get(keyId);
1887
+ if (request) request.recalledToken = recall.token;
1888
+ return;
1889
+ }
1890
+ delegation.recalled = true;
1891
+ if (delegation.holding > 0) {
1892
+ await new Promise<void>((resolve) => {
1893
+ // A holder that never releases must not hold the recall open past its own lease: the
1894
+ // delegation expires here on our clock, and the home outwaits that by skew anyway. The
1895
+ // timer is dropped by whichever side wins, so a delegation that drains promptly does not
1896
+ // retain one for the rest of its lease.
1897
+ const remaining = Math.max(0, delegation.expiresMono - this.#monotonic());
1898
+ const timer = setTimeout(resolve, remaining);
1899
+ timer.unref?.();
1900
+ (delegation.drained ||= []).push(() => {
1901
+ clearTimeout(timer);
1902
+ resolve();
1903
+ });
1904
+ });
1905
+ }
1906
+ // The swap may have landed during the drain. The delegation OBJECT moved to the successor, so
1907
+ // surrendering here would revoke through an index this coordinator no longer owns and leave the
1908
+ // successor still holding a delegation whose release has already been written.
1909
+ await this.#authority().#surrender(keyId, delegation);
1910
+ }
1911
+
1912
+ /** Expire delegations and grants whose deadlines have passed. Bounded work per call. */
1913
+ tick(): void {
1914
+ const now = this.#monotonic();
1915
+ // A second, weaker signal than the incarnation: it catches a thread that stopped coordinating
1916
+ // without anything else starting, which advances no incarnation. Rate-limited well below the
1917
+ // tick because it cannot matter faster than a delegation lease, and it costs nothing at all on a
1918
+ // coordinator holding nothing — that one has already stopped ticking.
1919
+ if (now - this.#lastOwnershipPollMono >= OWNERSHIP_POLL_MS) {
1920
+ this.#lastOwnershipPollMono = now;
1921
+ this.#observeOwnership();
1922
+ }
1923
+ this.#pruneRemoteAdmissions(now);
1924
+ this.#prunePendingRemoteRevokes(now);
1925
+ // The budget counts EXPIRIES, not entries examined. Spending it on live entries would let a
1926
+ // table with more than a budget's worth of continuously renewed grants starve every expired
1927
+ // one behind them, and an uncollected expired grant answers `contended` to every other node.
1928
+ let budget = MAX_EXPIRIES_PER_TICK;
1929
+ for (const [keyId, delegation] of this.#delegations) {
1930
+ if (budget <= 0) break;
1931
+ if (delegation.expiresMono > now) continue;
1932
+ budget--;
1933
+ // A delegate may drop early; the handles it admitted are fenced by their own lease, which
1934
+ // never outlives the delegation that admitted them.
1935
+ this.#delegations.delete(keyId);
1936
+ this.#revokeAll(delegation);
1937
+ const waiters = delegation.drained;
1938
+ if (waiters) {
1939
+ delegation.drained = undefined;
1940
+ for (const resolve of waiters) resolve();
1941
+ }
1942
+ }
1943
+ budget = MAX_EXPIRIES_PER_TICK;
1944
+ for (const [keyId, grant] of this.#grants) {
1945
+ if (budget <= 0) break;
1946
+ // A home may NEVER forget a grant before its expiry — that asymmetry is the safety rule.
1947
+ if (grant.expiresMono > now) continue;
1948
+ budget--;
1949
+ this.#expireGrant(keyId, grant);
1950
+ }
1951
+ if (
1952
+ this.#delegations.size === 0 &&
1953
+ this.#grants.size === 0 &&
1954
+ !this.#remoteAdmissions &&
1955
+ !this.#pendingRemoteRevokes
1956
+ )
1957
+ tickingCoordinators.delete(this);
1958
+ }
1959
+
1960
+ /**
1961
+ * One table's half of `quiesceDelegations`. Never throws for a single grant: a transition needs to
1962
+ * know exactly what is still live, and one unreachable delegate must not hide the rest.
1963
+ */
1964
+ async quiesce(result: QuiesceResult, remaining: () => number): Promise<void> {
1965
+ if (this.#closed) return;
1966
+ // Delegate side first: this is what admits, and surrendering is purely local — it cannot be
1967
+ // refused by an unreachable peer, so it succeeds even when the recalls below do not.
1968
+ for (const delegation of [...this.#delegations.values()]) {
1969
+ try {
1970
+ await withDeadline(this.onDelegationRecall({ key: delegation.key, token: delegation.token }), remaining());
1971
+ result.surrendered++;
1972
+ } catch (error) {
1973
+ result.outstanding.push({
1974
+ table: this.table,
1975
+ key: delegation.key,
1976
+ reason: `this node still holds a delegation it could not drain: ${(error as Error)?.message ?? error}`,
1977
+ });
1978
+ }
1979
+ }
1980
+ // Home side: tell delegates elsewhere to stop. `#beginRecall` owns the retry and confirmation
1981
+ // bookkeeping; this only drives it and reports what it did not confirm.
1982
+ for (const [keyId, grant] of [...this.#grants]) {
1983
+ if (grant.recallConfirmed) {
1984
+ result.recalled++;
1985
+ continue;
1986
+ }
1987
+ try {
1988
+ this.#beginRecall(keyId, grant);
1989
+ if (grant.recalling) await withDeadline(grant.recalling, remaining());
1990
+ if (grant.recallConfirmed) result.recalled++;
1991
+ else
1992
+ result.outstanding.push({
1993
+ table: this.table,
1994
+ key: grant.key,
1995
+ delegate: grant.delegate,
1996
+ reason: 'the delegate did not confirm it stopped admitting',
1997
+ });
1998
+ } catch (error) {
1999
+ result.outstanding.push({
2000
+ table: this.table,
2001
+ key: grant.key,
2002
+ delegate: grant.delegate,
2003
+ reason: `recall failed: ${(error as Error)?.message ?? error}`,
2004
+ });
2005
+ }
2006
+ }
2007
+ }
2008
+
2009
+ /**
2010
+ * Stop this coordinator and invalidate what it issued. Expiring every delegation is the part that
2011
+ * matters: `close()` runs when a transport is replaced (a component reload is enough), and a
2012
+ * successor coordinator must not be able to grant a key whose predecessor handles are still live.
2013
+ */
2014
+ close(): void {
2015
+ this.#closed = true;
2016
+ liveCoordinators.delete(this);
2017
+ // State that was handed to a successor is that coordinator's now; expiring it here would
2018
+ // invalidate delegations the successor is correctly still honouring.
2019
+ if (this.#handedOff) {
2020
+ tickingCoordinators.delete(this);
2021
+ return;
2022
+ }
2023
+ for (const delegation of this.#delegations.values()) {
2024
+ delegation.recalled = true;
2025
+ delegation.expiresMono = -Infinity;
2026
+ this.#revokeAll(delegation);
2027
+ const waiters = delegation.drained;
2028
+ if (waiters) {
2029
+ delegation.drained = undefined;
2030
+ for (const resolve of waiters) resolve();
2031
+ }
2032
+ }
2033
+ for (const pending of this.#pendingDelegations.values()) pending.markRecalled();
2034
+ // Remote admissions (harper-pro#852) that were NOT handed to a successor: this thread is going
2035
+ // away, so fence their handles (fail closed) and tell the owner it may give the delegation up now
2036
+ // rather than hold it for a full lease waiting on a worker that has gone.
2037
+ if (this.#remoteAdmissions) {
2038
+ for (const remote of this.#remoteAdmissions.values()) {
2039
+ fireRevokeAndForget(remote.revoke);
2040
+ this.#settleFenceWaiters(remote);
2041
+ this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
2042
+ }
2043
+ this.#remoteAdmissions = undefined;
2044
+ this.#remoteByOwnerId = undefined;
2045
+ }
2046
+ // Any ack still waiting on an entry that never installed cannot be honored here; resolve it so the
2047
+ // owner's revoke wait does not hang on a thread that is gone (its own lease bounds it regardless).
2048
+ if (this.#pendingRemoteRevokes) {
2049
+ for (const entry of this.#pendingRemoteRevokes.values()) for (const resolve of entry.resolvers) resolve();
2050
+ this.#pendingRemoteRevokes = undefined;
2051
+ }
2052
+ // The delegations THIS node issued as a home outlive it: no peer sees a local close, so each one
2053
+ // stands until its own deadline. Leave the latest of those deadlines, and the counter, for
2054
+ // whatever coordinator takes this table next.
2055
+ let grantableAfterMono = -Infinity;
2056
+ for (const grant of this.#grants.values()) {
2057
+ // A grant to THIS node is settled by the same close: the loop above revoked the delegation it
2058
+ // authorized. Only what another node holds outlives this coordinator unseen.
2059
+ if (grant.delegate === this.nodeId) continue;
2060
+ if (grant.expiresMono > grantableAfterMono) grantableAfterMono = grant.expiresMono;
2061
+ }
2062
+ const retirementKey = this.#retirementKey();
2063
+ const retired = retiredCoordinators.get(retirementKey);
2064
+ retiredCoordinators.set(retirementKey, {
2065
+ grantableAfterMono: Math.max(retired?.grantableAfterMono ?? -Infinity, grantableAfterMono),
2066
+ counter: Math.max(retired?.counter ?? 0, this.#counter),
2067
+ });
2068
+ this.#delegations.clear();
2069
+ this.#pendingDelegations.clear();
2070
+ this.#pendingRequests.clear();
2071
+ this.#grants.clear();
2072
+ this.#grantsByRequester.clear();
2073
+ this.#dependencySets.clear();
2074
+ this.#everDelegated.clear();
2075
+ tickingCoordinators.delete(this);
2076
+ }
2077
+
2078
+ /**
2079
+ * Ownership as this thread can actually observe it. Only a LOSS is recorded: regaining it is the
2080
+ * grant path's business, because that is where the incarnation is available to say whether anything
2081
+ * else coordinated in between.
2082
+ */
2083
+ #observeOwnership(): void {
2084
+ let owns: boolean;
2085
+ try {
2086
+ owns = this.transport.ownsCoordination();
2087
+ } catch {
2088
+ // A transport that cannot answer is not proof this thread kept coordinating, and the whole
2089
+ // point of the horizon is what happened while it did not.
2090
+ owns = false;
2091
+ }
2092
+ if (!owns) this.#ownedSinceMono = undefined;
2093
+ }
2094
+
2095
+ /**
2096
+ * The horizon owning coordination imposes, anchored at the instant this coordinator STARTED
2097
+ * coordinating under the incarnation it is granting under — see `#ownedSinceMono`.
2098
+ *
2099
+ * `homeIncarnation` is the signal, not `ownsCoordination()`. Sampling a boolean proves the answer
2100
+ * at the instant it is read and never that ownership was unbroken between two reads, so ownership
2101
+ * alternating faster than the sample interval could alias away entirely. §5.1 makes the incarnation
2102
+ * advance once per COORDINATION incarnation — that is what keeps the fencing token orderable — so a
2103
+ * value this coordinator has not granted under is the transport stating that something else
2104
+ * coordinated for this node, whatever the boolean said in between.
2105
+ */
2106
+ #ownershipHorizon(now: number, homeIncarnation: number): number {
2107
+ if (this.#coordinatingIncarnation !== homeIncarnation || this.#ownedSinceMono === undefined) {
2108
+ this.#loseFreshnessHistory();
2109
+ // Ownership STARTED here, which is the one thing the waiver cannot cover.
2110
+ // `grantableAfterMono` attests that no previous INCARNATION OF THIS PROCESS had issued
2111
+ // delegations — a cold start, a fresh database, a test — and says nothing about the sibling
2112
+ // thread that was coordinating until this instant. The constructor records both directly, so
2113
+ // a coordinator that has coordinated since it was built never reaches this and keeps the
2114
+ // waiver; every other way of arriving at ownership lands here and loses it.
2115
+ this.#coordinatingIncarnation = homeIncarnation;
2116
+ this.#quarantineWaived = false;
2117
+ this.#ownedSinceMono = now;
2118
+ }
2119
+ if (this.#quarantineWaived) return -Infinity;
2120
+ return this.#ownedSinceMono + DELEGATION_LEASE_MS + this.#skewMs;
2121
+ }
2122
+
2123
+ /** Fail closed on a generation older than one already ACTED on here (see `highestGeneration`). */
2124
+ #generationIsCurrent(generation: number): boolean {
2125
+ const highest = highestGeneration.get(this.database);
2126
+ return highest === undefined || generation >= highest;
2127
+ }
2128
+
2129
+ /**
2130
+ * Raise the floor, at the moment authority is actually taken under this generation — a token
2131
+ * minted or a delegation installed — and never merely on reading a map.
2132
+ *
2133
+ * Observing was the obvious place and it is the wrong one: a single `homeMap()` that returns a
2134
+ * too-large generation once, from a partial publish or a transport glitch, would pin the floor
2135
+ * above anything the operator ever publishes and fail every later lock on this database until the
2136
+ * thread restarts. Nothing was minted under that reading, so nothing needs protecting from it. The
2137
+ * invariant only ever needed to be "never mint below a generation already minted".
2138
+ */
2139
+ #recordGenerationActedOn(generation: number): void {
2140
+ const highest = highestGeneration.get(this.database);
2141
+ if (highest === undefined || generation > highest) highestGeneration.set(this.database, generation);
2142
+ }
2143
+
2144
+ #prepareFreshnessGeneration(generation: number): void {
2145
+ if (this.#freshnessGeneration === undefined) {
2146
+ this.#freshnessGeneration = generation;
2147
+ return;
2148
+ }
2149
+ if (this.#freshnessGeneration === generation) return;
2150
+ this.#freshnessGeneration = generation;
2151
+ this.#loseFreshnessHistory();
2152
+ }
2153
+
2154
+ #loseFreshnessHistory(): void {
2155
+ this.#dependencySets.clear();
2156
+ this.#everDelegated.clear();
2157
+ this.#trustVirginKeys = false;
2158
+ }
2159
+
2160
+ #freshnessFor(keyId: unknown): LockDependencySet | null {
2161
+ const retained = this.#dependencySets.get(keyId);
2162
+ if (retained) {
2163
+ this.#dependencySets.delete(keyId);
2164
+ this.#dependencySets.set(keyId, retained);
2165
+ return retained;
2166
+ }
2167
+ return this.#trustVirginKeys && !this.#everDelegated.has(keyId) ? [] : null;
2168
+ }
2169
+
2170
+ #rememberDependencies(keyId: unknown, dependencies: LockDependencySet): void {
2171
+ this.#dependencySets.delete(keyId);
2172
+ this.#dependencySets.set(keyId, dependencies);
2173
+ while (this.#dependencySets.size > MAX_DEPENDENCY_SETS_PER_TABLE)
2174
+ this.#dependencySets.delete(this.#dependencySets.keys().next().value);
2175
+ }
2176
+
2177
+ #retirementKey(): string {
2178
+ return `${this.database}\u0000${this.table}`;
2179
+ }
2180
+
2181
+ #ringKey(keyId: unknown): string {
2182
+ return ringKeyFor(this.database, this.table, keyId);
2183
+ }
2184
+
2185
+ /** The coordinator holding this node's authority now: this one, or the end of the handoff chain. */
2186
+ #authority(): LockCoordinator {
2187
+ let coordinator: LockCoordinator = this;
2188
+ while (coordinator.#successor) coordinator = coordinator.#successor;
2189
+ return coordinator;
2190
+ }
2191
+
2192
+ #liveDelegation(keyId: unknown, leaseMs: number, generation: number): Delegation | undefined {
2193
+ const delegation = this.#delegations.get(keyId);
2194
+ if (!delegation || delegation.recalled) return undefined;
2195
+ // A delegation is authority within ONE generation. After a generation change the key may have
2196
+ // been re-homed, and the new home knows nothing of this token — so keeping it would let this
2197
+ // node admit alongside whoever the new home grants.
2198
+ if (delegation.token[0] !== generation) {
2199
+ // Authority is gone, not merely stale: the key may have been re-homed to a node that knows
2200
+ // nothing of this token. Forgetting the delegation without revoking would leave its handles
2201
+ // able to commit alongside whatever the new home grants.
2202
+ this.#delegations.delete(keyId);
2203
+ this.#revokeAll(delegation);
2204
+ return undefined;
2205
+ }
2206
+ // The admission may not outlive the delegation that admitted it, so a delegation without room
2207
+ // for the whole lease is renewed rather than stretched.
2208
+ if (delegation.expiresMono - this.#monotonic() < leaseMs) return undefined;
2209
+ return delegation;
2210
+ }
2211
+
2212
+ #admit(delegation: Delegation, leaseMs: number): LockRound {
2213
+ const mintedMono = this.#monotonic();
2214
+ // Admissions that unlocked are kept only until their handle's own lease fences it. Collecting
2215
+ // them here rather than on a timer keeps the work on the path that creates it, and a delegation
2216
+ // can accumulate at most one entry per overlapping lock in its own lease.
2217
+ this.#pruneAdmissions(delegation, mintedMono);
2218
+ const admissionId = this.#nextAdmissionId++;
2219
+ // The entry exists from the instant of admission, so a recall between admit and register still
2220
+ // sees it.
2221
+ delegation.admissions.set(admissionId, { revoke: noRevoke, expiresMono: mintedMono + leaseMs, holding: true });
2222
+ delegation.holding++;
2223
+ this.#admissions.set(admissionId, delegation);
2224
+ return { tsR: this.#nextTimestamp(), mintedMono, admissionId };
2225
+ }
2226
+
2227
+ /**
2228
+ * Drop admissions whose handle's lease has run out; they can no longer commit anything.
2229
+ *
2230
+ * Insertion order is monotonic order, not expiry order: under one lease length the leading run IS
2231
+ * the expired set, but a longer-lease admission at the head hides every shorter one behind it for
2232
+ * its own remaining lease. Scanning the whole map on every admission instead would be quadratic on
2233
+ * the hot key this design exists to make cheap, so it is swept only once it has outgrown the live
2234
+ * set the previous sweep measured — amortized O(1) per admission, and the map stays within twice
2235
+ * the live set rather than growing at the lock rate.
2236
+ *
2237
+ * Never has a drain to wake: `#admit` reaches neither call site for a recalled delegation.
2238
+ */
2239
+ #pruneAdmissions(delegation: Delegation, now: number): void {
2240
+ for (const [admissionId, admission] of delegation.admissions) {
2241
+ if (admission.expiresMono > now) break;
2242
+ this.#dropAdmission(delegation, admissionId, admission);
2243
+ }
2244
+ if (delegation.admissions.size < delegation.sweepAtSize) return;
2245
+ for (const [admissionId, admission] of delegation.admissions)
2246
+ if (admission.expiresMono <= now) this.#dropAdmission(delegation, admissionId, admission);
2247
+ delegation.sweepAtSize = delegation.admissions.size * 2 + ADMISSION_SWEEP_FLOOR;
2248
+ }
2249
+
2250
+ #dropAdmission(delegation: Delegation, admissionId: number, admission: Admission): void {
2251
+ delegation.admissions.delete(admissionId);
2252
+ this.#admissions.delete(admissionId);
2253
+ if (admission.holding) delegation.holding--;
2254
+ }
2255
+
2256
+ /**
2257
+ * Register how to revoke the handle an admission produced. `Table.lock()` calls this once the
2258
+ * handle has joined the round, so a recall can fence a write that was staged and then unlocked.
2259
+ */
2260
+ registerAdmission(admissionId: number, revoke: () => void | Promise<void>): void {
2261
+ const admission = this.#admissions.get(admissionId)?.admissions.get(admissionId);
2262
+ if (admission) {
2263
+ admission.revoke = revoke;
2264
+ return;
2265
+ }
2266
+ // A remote admission (harper-pro#852): the delegation lives on the owner, the handle here. A revoke
2267
+ // that already latched (it beat this registration) fences the handle now and releases the ack that
2268
+ // was waiting on the fence; otherwise store the revoker for a later recall.
2269
+ const remote = this.#remoteAdmissions?.get(admissionId);
2270
+ if (remote) {
2271
+ remote.revoke = revoke;
2272
+ if (remote.revoked) {
2273
+ // A revoke latched before this registration: fence now, then drop the entry and release the
2274
+ // ack that was waiting on the fence. `#dropRemoteAdmission` is what resolves that ack, so an
2275
+ // async revoker must settle FIRST — resolving it early would tell the owner this handle is
2276
+ // fenced while it still is not, and the owner would write the release over a live writer. A fence
2277
+ // that FAILS settles nothing: the entry is left for the lease sweep, which resolves the ack once
2278
+ // the handle is provably dead. The failure is warned about inside and has nobody here to go to,
2279
+ // so it is absorbed rather than left to Node's unhandled-rejection policy.
2280
+ const fenced = this.#fenceRemoteAdmission(admissionId, remote);
2281
+ if (isPromiseLike(fenced)) fenced.catch(() => {});
2282
+ }
2283
+ return;
2284
+ }
2285
+ // The admission was already revoked or collected between admit and register — the handle has no
2286
+ // authority to keep, so revoke it now (safely: a relay revoker collected in this window is async
2287
+ // and could reject on a dead port) rather than leaving it unfenced.
2288
+ fireRevokeAndForget(revoke);
2289
+ }
2290
+
2291
+ /**
2292
+ * Install a granted delegation, with its deadline anchored at the moment the request was SENT
2293
+ * rather than at the moment the reply arrived. The home started its own clock when it granted, so
2294
+ * anchoring on arrival would hand a delayed reply more time than the home is holding the key for —
2295
+ * and a reply delayed past the whole delegation must be discarded, not installed (§5.2).
2296
+ */
2297
+ #installDelegation(
2298
+ keyId: unknown,
2299
+ key: any,
2300
+ token: FencingToken,
2301
+ leaseMs: number,
2302
+ requestedAtMono: number,
2303
+ dependencies: LockDependencySet
2304
+ ): Delegation | undefined {
2305
+ const expiresMono = requestedAtMono + leaseMs;
2306
+ if (expiresMono <= this.#monotonic()) return undefined;
2307
+ // The delegate side of taking authority under a generation: installing this makes it admit under
2308
+ // `token[0]`, so that is the floor a later map may not go below.
2309
+ this.#recordGenerationActedOn(token[0]);
2310
+ const existing = this.#delegations.get(keyId);
2311
+ // A reply that lost a race with a newer delegation for the same key must not move it backwards.
2312
+ if (existing && compareTokens(existing.token, token) >= 0) return existing;
2313
+ if (existing && !existing.recalled) {
2314
+ // A RENEWAL of authority this node never lost. Advance the token and the deadline in place so
2315
+ // the admissions it is still answerable for ride along; installing a fresh object here
2316
+ // orphaned them, and the next recall then surrendered while a live handle could still commit.
2317
+ existing.token = token;
2318
+ existing.expiresMono = expiresMono;
2319
+ this.#startTicking();
2320
+ return existing;
2321
+ }
2322
+ // Replacing a recalled delegation: its handles were revoked at surrender, so nothing carries.
2323
+ if (existing) this.#revokeAll(existing);
2324
+ const delegation: Delegation = {
2325
+ key,
2326
+ token,
2327
+ dependencies,
2328
+ expiresMono,
2329
+ recalled: false,
2330
+ admissions: new Map(),
2331
+ holding: 0,
2332
+ sweepAtSize: ADMISSION_SWEEP_FLOOR,
2333
+ };
2334
+ this.#delegations.set(keyId, delegation);
2335
+ this.#startTicking();
2336
+ return delegation;
2337
+ }
2338
+
2339
+ async #establishFreshness(
2340
+ key: any,
2341
+ requirement: LockDependencySet | null,
2342
+ homes: readonly string[],
2343
+ deadlineMono: number,
2344
+ pending?: PendingDelegation
2345
+ ): Promise<LockDependencySet> {
2346
+ if (pending?.recalled) throw PENDING_DELEGATION_RECALLED;
2347
+ if (requirement?.length === 0) return requirement;
2348
+ const remaining = deadlineMono - this.#monotonic();
2349
+ if (remaining <= 0) throw new Error('the lock wait elapsed before its freshness barrier started');
2350
+ const established = Promise.resolve().then(() =>
2351
+ this.transport.establishLockFreshness(this.database, this.table, key, requirement, remaining)
2352
+ );
2353
+ const timeout = delay(remaining);
2354
+ established.then(
2355
+ () => timeout.cancel(),
2356
+ () => timeout.cancel()
2357
+ );
2358
+ const alternatives: Promise<LockDependencySet | void>[] = [
2359
+ established,
2360
+ timeout.promise.then(() => {
2361
+ throw new Error('the lock wait elapsed before its freshness barrier completed');
2362
+ }),
2363
+ ];
2364
+ if (pending)
2365
+ alternatives.push(
2366
+ pending.recalledPromise.then(() => {
2367
+ timeout.cancel();
2368
+ throw PENDING_DELEGATION_RECALLED;
2369
+ })
2370
+ );
2371
+ const result = await Promise.race(alternatives);
2372
+ if (requirement !== null) return requirement;
2373
+ const recovered = normalizeDependencies(result);
2374
+ if (!recovered) throw new Error('the recovery barrier returned no usable applied-position set');
2375
+ const allowedOrigins = new Set(homes);
2376
+ return recovered.filter(([origin]) => allowedOrigins.has(origin));
2377
+ }
2378
+
2379
+ /** This node is the key's home: grant to itself through exactly the same table a peer would use. */
2380
+ #grantLocally(keyId: unknown, key: any, homeMap: LockHomeMap, leaseMs: number): DelegationReply {
2381
+ return this.#grant(keyId, key, homeMap, leaseMs, this.nodeId);
2382
+ }
2383
+
2384
+ async #requestRemotely(
2385
+ home: string,
2386
+ keyId: unknown,
2387
+ key: any,
2388
+ homeMap: LockHomeMap,
2389
+ leaseMs: number,
2390
+ deadlineMono: number
2391
+ ): Promise<DelegationReply> {
2392
+ try {
2393
+ // A half-open connection to the home would otherwise leave `lock()` pending forever, past
2394
+ // its own timeout and past the native lease — and a reply arriving after that lease has
2395
+ // fired cannot be joined to the handle anyway.
2396
+ const remaining = Math.max(1, deadlineMono - this.#monotonic());
2397
+ let raced = false;
2398
+ const requested = Promise.resolve(
2399
+ this.transport.requestDelegation(home, this.database, this.table, {
2400
+ key,
2401
+ requester: this.nodeId,
2402
+ generation: homeMap.generation,
2403
+ leaseMs,
2404
+ })
2405
+ );
2406
+ // A reply that arrives after we stopped waiting still granted us the key on the home, which
2407
+ // would then hold it for the whole delegation while every other node is denied. Hand it back.
2408
+ // Through the authority, not through this object: a transport swap can land while the reply
2409
+ // is in flight, and `handOffTo` empties this coordinator's delegations. The handback's
2410
+ // "not while a delegation for the key is held" guard would then read an empty map and give
2411
+ // back a grant that still backs the successor's live delegation.
2412
+ requested.then(
2413
+ (reply) => {
2414
+ if (raced && reply?.granted && reply.token) this.#authority().#releaseUnclaimedGrant(key, reply.token);
2415
+ },
2416
+ () => {}
2417
+ );
2418
+ const timeout = delay(remaining);
2419
+ // Dropped once the race settles: a reply that beats the timeout would otherwise leave a timer
2420
+ // holding this closure for the caller's whole remaining wait.
2421
+ requested.then(
2422
+ () => timeout.cancel(),
2423
+ () => timeout.cancel()
2424
+ );
2425
+ return await Promise.race([
2426
+ requested,
2427
+ timeout.promise.then(() => {
2428
+ raced = true;
2429
+ return { granted: false, reason: 'timeout', retryAfterMs: 0 } as DelegationReply;
2430
+ }),
2431
+ ]);
2432
+ } catch (error) {
2433
+ // An unreachable home blocks only the keys it homes, which is the availability property the
2434
+ // whole design exists for — it is not a reason to admit without one.
2435
+ throw new LockUnavailableError(
2436
+ `Could not reach ${home}, the home node for this key on ${this.database}.${this.table}: ${(error as Error)?.message ?? error}`
2437
+ );
2438
+ }
2439
+ }
2440
+
2441
+ /**
2442
+ * Give back a delegation this node asked for but stopped waiting on. Without it the home holds the
2443
+ * key for a delegation nobody is using, and every other node is denied for its full duration.
2444
+ */
2445
+ #releaseUnclaimedGrant(key: any, token: FencingToken): Promise<void> | void {
2446
+ const keyId = this.#keyIdOf(key);
2447
+ const held = this.#delegations.get(keyId);
2448
+ // ANY live delegation for this key means this node is using it, and the tokens need not match.
2449
+ // A duplicate or delayed request from this node renews the home's grant IN PLACE (`#grant`'s
2450
+ // renewal branch mutates `existing.token`), so the home can hold a newer token than the one we
2451
+ // installed. Releasing that token would clear the grant still backing our own live delegation
2452
+ // and let the home hand the key to another node while we are inside it. Comparing tokens here
2453
+ // caught only the case where we installed this exact grant.
2454
+ if (held) return;
2455
+ return this.#writeControlSafely({ type: 'lockRelease', key, requester: this.nodeId, token, dependencies: null });
2456
+ }
2457
+
2458
+ #grant(keyId: unknown, key: any, homeMap: LockHomeMap, leaseMs: number, requester: string): DelegationReply {
2459
+ const now = this.#monotonic();
2460
+ this.#prepareFreshnessGeneration(homeMap.generation);
2461
+ // Consulted unconditionally — it is what observes a gap in ownership, and the waiver it applies
2462
+ // to itself is the only part `grantableAfterMono` may switch off.
2463
+ const quarantine = Math.max(this.#grantableAfterMono, this.#ownershipHorizon(now, homeMap.homeIncarnation)) - now;
2464
+ // NOT `contended`: the quarantine runs for a full delegation lease, and `MAX_LOCK_TIMEOUT_MS` is
2465
+ // shorter than that, so retrying it would spend the caller's whole budget and then answer 423 —
2466
+ // "held by someone else" — for a key nobody holds.
2467
+ if (quarantine > 0) return { granted: false, reason: 'quarantine', retryAfterMs: Math.min(quarantine, 250) };
2468
+ const existing = this.#grants.get(keyId);
2469
+ if (existing) {
2470
+ // An expired grant is not a live one. Collecting it here rather than trusting `tick()` is
2471
+ // what keeps a table whose expiry budget is saturated from answering `contended` forever.
2472
+ if (existing.expiresMono <= now) this.#expireGrant(keyId, existing);
2473
+ else if (existing.recalling || existing.recallConfirmed)
2474
+ // A recall is in flight, or the delegate has already confirmed one. Renewing now — even for
2475
+ // the node being recalled — would mint a token its own release no longer matches, and the
2476
+ // contender would never get the key. `recallConfirmed` has to be here as well as in
2477
+ // `#beginRecall`: the delegate can confirm and re-ask before its release reaches the home,
2478
+ // and renewing then would leave a grant nothing will recall again and nothing can release.
2479
+ return { granted: false, reason: 'contended', retryAfterMs: 25 };
2480
+ else if (existing.delegate === requester) {
2481
+ // Renewal for the node that already holds it: extend rather than recall itself.
2482
+ this.#recordGenerationActedOn(homeMap.generation);
2483
+ existing.token = [homeMap.generation, homeMap.homeIncarnation, ++this.#counter];
2484
+ existing.renewed = true;
2485
+ existing.expiresMono = now + DELEGATION_LEASE_MS + this.#skewMs;
2486
+ return {
2487
+ granted: true,
2488
+ token: existing.token,
2489
+ leaseMs: DELEGATION_LEASE_MS,
2490
+ dependencies: existing.dependencies,
2491
+ };
2492
+ } else {
2493
+ // Someone else holds it. Start the recall and make the caller come back — holding the
2494
+ // request open across a drain would tie the home's reply to the previous delegate's
2495
+ // liveness.
2496
+ this.#beginRecall(keyId, existing);
2497
+ return { granted: false, reason: 'contended', retryAfterMs: 25 };
2498
+ }
2499
+ }
2500
+ if (this.#grants.size >= MAX_DELEGATIONS_PER_TABLE) return { granted: false, reason: 'capacity' };
2501
+ const perRequester = this.#grantsByRequester.get(requester) ?? 0;
2502
+ if (perRequester >= MAX_DELEGATIONS_PER_REQUESTER) return { granted: false, reason: 'capacity' };
2503
+ this.#recordGenerationActedOn(homeMap.generation);
2504
+ const token: FencingToken = [homeMap.generation, homeMap.homeIncarnation, ++this.#counter];
2505
+ const dependencies = this.#freshnessFor(keyId);
2506
+ this.#everDelegated.add(keyId);
2507
+ this.#grants.set(keyId, {
2508
+ key,
2509
+ delegate: requester,
2510
+ token,
2511
+ dependencies,
2512
+ // The home always outwaits the delegate by skew, so it cannot re-grant a key the previous
2513
+ // delegate still believes it holds. The delegation runs for its own fixed duration rather
2514
+ // than the caller's lock lease — a delegation sized to one lock leaves no room for the next
2515
+ // one, and every repeat lock would pay a round trip.
2516
+ expiresMono: now + DELEGATION_LEASE_MS + this.#skewMs,
2517
+ });
2518
+ this.#grantsByRequester.set(requester, perRequester + 1);
2519
+ this.#startTicking();
2520
+ return { granted: true, token, leaseMs: DELEGATION_LEASE_MS, dependencies };
2521
+ }
2522
+
2523
+ #beginRecall(keyId: unknown, grant: HomeGrant): void {
2524
+ if (grant.recalling || grant.recallConfirmed) return;
2525
+ if (grant.recallRetryAfterMono !== undefined && this.#monotonic() < grant.recallRetryAfterMono) return;
2526
+ if (grant.delegate === this.nodeId) {
2527
+ // We are both home and delegate. Recall ourselves through the same path a peer would take,
2528
+ // and settle it the same way: a recall that did not apply must leave the grant recallable,
2529
+ // or `recalling` stays latched here for the rest of the delegation and no contender ever
2530
+ // prompts another one.
2531
+ grant.recalling = this.onDelegationRecall({ key: grant.key, token: grant.token }).then(
2532
+ () => {
2533
+ grant.recalling = undefined;
2534
+ grant.recallConfirmed = true;
2535
+ },
2536
+ (error) => {
2537
+ warnOnce('failed to recall a local record lock delegation', error);
2538
+ grant.recalling = undefined;
2539
+ grant.recallRetryAfterMono = this.#monotonic() + RECALL_RETRY_MS;
2540
+ }
2541
+ );
2542
+ return;
2543
+ }
2544
+ // `.then`, not `Promise.resolve(recallDelegation(...))`: the transport call is evaluated before
2545
+ // `Promise.resolve` and can throw synchronously, which escapes this whole method — so the
2546
+ // handlers below never run, the retry interval is never armed, and the next contender pass
2547
+ // throws again immediately instead of backing off.
2548
+ grant.recalling = Promise.resolve()
2549
+ .then(() =>
2550
+ this.transport.recallDelegation(grant.delegate, this.database, this.table, {
2551
+ key: grant.key,
2552
+ token: grant.token,
2553
+ })
2554
+ )
2555
+ .then(() => {
2556
+ // The delegate confirmed it stopped admitting, so re-sending buys nothing: the grant is
2557
+ // cleared by its release entry, or failing that by its own deadline. Re-arming here is what
2558
+ // let a contender polling at 25 ms fire a recall per pass for the rest of the delegation.
2559
+ grant.recalling = undefined;
2560
+ grant.recallConfirmed = true;
2561
+ })
2562
+ .catch(() => {
2563
+ // An unreachable delegate is not a reason to re-grant early: the grant's own deadline is
2564
+ // what makes the successor safe, and it already includes the skew margin. Retrying IS
2565
+ // worthwhile here — the delegate may come back — but on its own interval, not the
2566
+ // contender's.
2567
+ grant.recalling = undefined;
2568
+ grant.recallRetryAfterMono = this.#monotonic() + RECALL_RETRY_MS;
2569
+ });
2570
+ }
2571
+
2572
+ /**
2573
+ * Give up a delegation: stop admitting, then write the release that lets the home re-grant. One
2574
+ * surrender per delegation, memoized: the home re-sends a recall `RECALL_RETRY_MS` after its own
2575
+ * recall call failed, which is far inside a delegation lease, so two recalls for the same token can
2576
+ * both be waiting on the drain and resume together. Without the memo the second finds the
2577
+ * admissions the first already emptied, has no fence to wait for, and writes the release while the
2578
+ * first is still waiting for a relayed handle to confirm it is fenced.
2579
+ */
2580
+ #surrender(keyId: unknown, delegation: Delegation): Promise<void> {
2581
+ return (delegation.surrendering ??= this.#surrenderOnce(keyId, delegation));
2582
+ }
2583
+
2584
+ async #surrenderOnce(keyId: unknown, delegation: Delegation): Promise<void> {
2585
+ if (this.#delegations.get(keyId) === delegation) this.#delegations.delete(keyId);
2586
+ // Capability, not just admission: anything this delegation admitted must be unable to commit
2587
+ // before the home is told it may re-grant. A handle admitted on another worker thread revokes
2588
+ // over a message, so wait for its fence to land — bounded by the handle's own lease, past which
2589
+ // it fences itself — before writing the release that lets the home re-grant the key.
2590
+ await this.#revokeAllAndSettle(delegation);
2591
+ const entry: LockReleaseEntry = {
2592
+ type: 'lockRelease',
2593
+ key: delegation.key,
2594
+ requester: this.nodeId,
2595
+ token: delegation.token,
2596
+ dependencies: delegation.dependencies,
2597
+ };
2598
+ return this.#writeControlSafely(entry);
2599
+ }
2600
+
2601
+ /**
2602
+ * Revoke every handle this delegation is answerable for and forget the admissions. Returns, per
2603
+ * admission, the revoker's outcome paired with the admission's own monotonic deadline, so a caller
2604
+ * that must not write the release before the fence is proven (`#surrender`) can bound its wait by
2605
+ * that deadline. Callers tearing the delegation down anyway (`tick`, `close`) ignore the result. A
2606
+ * revoke that throws synchronously is captured as a rejected outcome, never swallowed as fenced.
2607
+ */
2608
+ #revokeAll(delegation: Delegation): { outcome: void | Promise<void>; expiresMono: number }[] {
2609
+ const admissions = delegation.admissions;
2610
+ delegation.admissions = new Map();
2611
+ delegation.holding = 0;
2612
+ delegation.sweepAtSize = ADMISSION_SWEEP_FLOOR;
2613
+ const outcomes: { outcome: void | Promise<void>; expiresMono: number }[] = [];
2614
+ for (const [admissionId, admission] of admissions) {
2615
+ this.#admissions.delete(admissionId);
2616
+ let outcome: void | Promise<void>;
2617
+ try {
2618
+ outcome = admission.revoke();
2619
+ } catch (error) {
2620
+ // A synchronous throw is NOT a fence: surface it as a rejected outcome so `#revokeAllAndSettle`
2621
+ // waits out the admission's own lease rather than releasing against a handle that may still commit.
2622
+ outcome = Promise.reject(error);
2623
+ }
2624
+ // Every fire-and-forget caller (`tick`, `close`) discards this array, so a rejecting outcome —
2625
+ // a sync throw above, or a relayed revoker whose promise rejects on a dead sibling port — must
2626
+ // carry its own no-op handler here or Node's default policy would exit the worker. The awaited
2627
+ // path (`#revokeAllAndSettle`) still sees the rejection: a settled promise may be awaited again.
2628
+ if (outcome && typeof (outcome as Promise<void>).then === 'function') (outcome as Promise<void>).catch(() => {});
2629
+ outcomes.push({ outcome, expiresMono: admission.expiresMono });
2630
+ }
2631
+ return outcomes;
2632
+ }
2633
+
2634
+ /**
2635
+ * `#revokeAll`, then wait for every asynchronous fence (a handle on another worker acking its
2636
+ * `revokeLease`) before the caller writes the release. Each wait is bounded HERE by the admission's
2637
+ * own remaining lease, independent of what the transport's revoker does: a revoker that rejects,
2638
+ * throws, or never settles cannot make `#surrender` publish the release before the fenced handle's
2639
+ * lease has elapsed — past which the handle fences itself and can no longer commit. The transport's
2640
+ * own lease-bounded ack (harper-pro) is the fast path; this is the guarantee.
2641
+ */
2642
+ async #revokeAllAndSettle(delegation: Delegation): Promise<void> {
2643
+ const outcomes = this.#revokeAll(delegation);
2644
+ // ONE shared lease timer for the whole settle, not one per admission: a recall of a delegation
2645
+ // holding thousands of relayed admissions would otherwise arm thousands of timers at exactly the
2646
+ // moment it is handing off. The shared deadline is the LATEST lease among them, which bounds every
2647
+ // handle in the set — waiting past a shorter lease only ever errs towards holding the release
2648
+ // longer, never towards writing it early.
2649
+ let latestExpiry = -Infinity;
2650
+ let anyAsync = false;
2651
+ for (const { outcome, expiresMono } of outcomes) {
2652
+ if (!isPromiseLike(outcome)) continue;
2653
+ anyAsync = true;
2654
+ if (expiresMono > latestExpiry) latestExpiry = expiresMono;
2655
+ }
2656
+ // A synchronous revoker has already fenced by the time it returns, so it contributes nothing to
2657
+ // wait on: only the promise-returning ones cost an entry here.
2658
+ if (!anyAsync) return;
2659
+ const leaseTimer = delay(Math.max(0, latestExpiry - this.#monotonic()));
2660
+ const waits: Promise<unknown>[] = [];
2661
+ for (const { outcome } of outcomes) {
2662
+ if (!isPromiseLike(outcome)) continue;
2663
+ // Race the fence ack against the shared lease deadline: whichever comes first, the handle can no
2664
+ // longer commit once we return. A rejected fence is not a confirmed one, so it falls through to
2665
+ // the same deadline rather than resolving early.
2666
+ waits.push(
2667
+ Promise.race([
2668
+ outcome.catch((error) => {
2669
+ warnOnce('a record lock handle did not confirm revocation; waiting out its lease', error);
2670
+ return leaseTimer.promise;
2671
+ }),
2672
+ leaseTimer.promise,
2673
+ ])
2674
+ );
2675
+ }
2676
+ try {
2677
+ await Promise.all(waits);
2678
+ } finally {
2679
+ leaseTimer.cancel();
2680
+ }
2681
+ }
2682
+
2683
+ #clearGrant(keyId: unknown, grant: HomeGrant): void {
2684
+ if (this.#grants.get(keyId) !== grant) return;
2685
+ this.#grants.delete(keyId);
2686
+ const count = (this.#grantsByRequester.get(grant.delegate) ?? 1) - 1;
2687
+ if (count > 0) this.#grantsByRequester.set(grant.delegate, count);
2688
+ else this.#grantsByRequester.delete(grant.delegate);
2689
+ }
2690
+
2691
+ #expireGrant(keyId: unknown, grant: HomeGrant): void {
2692
+ if (this.#grants.get(keyId) !== grant) return;
2693
+ this.#dependencySets.delete(keyId);
2694
+ this.#clearGrant(keyId, grant);
2695
+ }
2696
+
2697
+ async #writeControlSafely(entry: LockReleaseEntry): Promise<void> {
2698
+ try {
2699
+ const position = await this.#writeControl(entry);
2700
+ this.#authority().applyEntry(entry, this.nodeId, typeof position === 'number' ? position : undefined);
2701
+ } catch (error) {
2702
+ // A lost release costs the key its remaining lease on the home; it never costs exclusion.
2703
+ warnOnce('failed to write a record lock release entry', error);
2704
+ // A local home can discard its own failed handback safely; its successor will recover.
2705
+ try {
2706
+ const keyId = this.#keyIdOf(entry.key);
2707
+ const grant = this.#grants.get(keyId);
2708
+ if (grant?.delegate === this.nodeId && compareTokens(grant.token, entry.token) === 0)
2709
+ this.#expireGrant(keyId, grant);
2710
+ } catch {}
2711
+ }
2712
+ }
2713
+
2714
+ #startTicking() {
2715
+ if (!this.#autoTick) return;
2716
+ tickingCoordinators.add(this);
2717
+ ensureTicking();
2718
+ }
2719
+ }
2720
+
2721
+ /**
2722
+ * What ended an exhausted `acquire()`, for the 503 it throws. A `timeout` is not phrased as a home
2723
+ * answer: it is this node's own deadline, and an operator told "the home answered timeout" looks for
2724
+ * a fault on a node that was simply not waited for.
2725
+ */
2726
+ function describeExhaustedWait(terminal: DelegationReply | undefined, hasCurrentMap: boolean): string {
2727
+ if (terminal)
2728
+ return terminal.reason === 'timeout'
2729
+ ? "no reply from the key's home within the wait"
2730
+ : `the key's home answered ${terminal.reason ?? (terminal.granted ? 'grants that arrived too late to use' : 'nothing usable')}`;
2731
+ return hasCurrentMap
2732
+ ? 'the record lock home map changed generation before the wait ended'
2733
+ : 'this node has no current record lock home map';
2734
+ }
2735
+
2736
+ /** A sleep whose timer the winner of a race can drop, rather than let it run out its full delay. */
2737
+ function delay(ms: number): { promise: Promise<void>; cancel: () => void } {
2738
+ let timer: ReturnType<typeof setTimeout>;
2739
+ const promise = new Promise<void>((resolve) => {
2740
+ timer = setTimeout(resolve, ms);
2741
+ timer.unref?.();
2742
+ });
2743
+ return { promise, cancel: () => clearTimeout(timer!) };
2744
+ }
2745
+
2746
+ const clusterLockTransports = new Map<string, ClusterLockTransport>();
2747
+ // Databases that have had a transport registered on this worker. A later absence is a transport
2748
+ // that went away — a component reload, a failed reconnect — not proof the node became standalone,
2749
+ // so cluster scope must keep failing closed rather than quietly reverting to a node-local lock.
2750
+ const clusterRequiredDatabases = new Set<string>();
2751
+ type CoordinatorResolver = (database: string, table: string) => LockCoordinator | undefined;
2752
+ let coordinatorResolver: CoordinatorResolver | undefined;
2753
+ // Applying a release is bookkeeping on this node's own grant table and needs no transport, so it
2754
+ // resolves the coordinator that HOLDS the grant rather than the one a registered transport answers
2755
+ // for. Dropping a peer's clean-handoff release during a reconnect denies every other node that key
2756
+ // for the delegation's whole deadline. Requests and recalls keep the transport-gated resolver: both
2757
+ // are answers to a live peer, and failing them closed while the transport is gone is the right shape.
2758
+ let admittingResolver: CoordinatorResolver | undefined;
2759
+ type ControlWriter = (entry: LockControlEntry) => Promise<number | void> | number | void;
2760
+ type ControlWriterResolver = (database: string, table: string) => ControlWriter | undefined;
2761
+ let controlWriterResolver: ControlWriterResolver | undefined;
2762
+
2763
+ /** Installed by Table.ts so a transport can push received entries in without importing Table. */
2764
+ export function setLockCoordinatorResolver(
2765
+ resolve: CoordinatorResolver,
2766
+ resolveAdmitting: CoordinatorResolver = resolve,
2767
+ resolveControlWriter?: ControlWriterResolver
2768
+ ) {
2769
+ coordinatorResolver = resolve;
2770
+ admittingResolver = resolveAdmitting;
2771
+ controlWriterResolver = resolveControlWriter;
2772
+ }
2773
+
2774
+ export interface QuiesceOutstanding {
2775
+ table: string;
2776
+ key: unknown;
2777
+ /** Present when this node was the HOME and the delegate did not confirm. */
2778
+ delegate?: string;
2779
+ reason: string;
2780
+ }
2781
+
2782
+ export interface QuiesceResult {
2783
+ /**
2784
+ * Whether this result is a PROOF of quiescence, or merely a report of what was swept.
2785
+ *
2786
+ * A sweep can only visit coordinators that exist on this thread, and they are built lazily — a
2787
+ * table nothing has touched since a restart has none, so an empty `outstanding` would otherwise
2788
+ * read as "nothing is live" when the previous incarnation's delegations are still running
2789
+ * elsewhere. An orchestrator must require `complete && outstanding.length === 0`; anything else
2790
+ * means fall back to the drain interval for this node.
2791
+ */
2792
+ complete: boolean;
2793
+ /** Delegations this node held and gave up, so it can no longer admit under them. */
2794
+ surrendered: number;
2795
+ /** Grants this node issued whose delegate confirmed it stopped admitting. */
2796
+ recalled: number;
2797
+ /**
2798
+ * What is still live, or unprovable. Empty means this node is provably quiesced for the database —
2799
+ * and ONLY then, which is why a coordinator still inside its restart quarantine contributes an
2800
+ * entry here rather than reporting a clean sweep it cannot back.
2801
+ */
2802
+ outstanding: QuiesceOutstanding[];
2803
+ }
2804
+
2805
+ /**
2806
+ * Stop this node admitting under the current generation for `database`, and say whether it is
2807
+ * provably done (harper-pro#856).
2808
+ *
2809
+ * A membership change otherwise has to wait out `DELEGATION_LEASE_MS + LOCK_LEASE_SKEW_MS` before the
2810
+ * next generation may activate, because authority already issued under the old one has to expire —
2811
+ * roughly six minutes during which the staged nodes serve no cluster locks at all. That interval is a
2812
+ * TIMER, chosen because you cannot recall what you cannot reach. In a planned transition every
2813
+ * participant is reachable, so the same guarantee can be *established* instead of waited out: this
2814
+ * drains both directions and reports what, if anything, is left.
2815
+ *
2816
+ * - **As a delegate** it surrenders every delegation it holds. This is the load-bearing half: a
2817
+ * delegate is what admits, and `onDelegationRecall` is the existing path whose resolution means
2818
+ * "nothing on this node can admit on that token" — live critical sections are drained, not cut.
2819
+ * - **As a home** it recalls every grant it issued, so delegates elsewhere stop too. Redundant when
2820
+ * every node is quiescing at once, and the reason this still terminates when one is not.
2821
+ *
2822
+ * `outstanding` empty on every node in `homes(g) ∪ homes(g+1)` is the operator's evidence that the
2823
+ * next generation may be activated immediately. Anything left is a node to fall back to the timer for,
2824
+ * or to fence externally — this never claims a drain it did not get, and never throws for one grant.
2825
+ */
2826
+ export async function quiesceDelegations(database: string, budgetMs: number): Promise<QuiesceResult> {
2827
+ const result: QuiesceResult = { complete: true, surrendered: 0, recalled: 0, outstanding: [] };
2828
+ const coordinators = [...liveCoordinators].filter((coordinator) => coordinator.database === database);
2829
+ // A DURATION, not an absolute deadline: the whole sweep gets this long, measured from here.
2830
+ const deadline = Date.now() + Math.max(0, budgetMs);
2831
+ const remaining = () => Math.max(0, deadline - Date.now());
2832
+ for (const coordinator of coordinators) {
2833
+ await coordinator.quiesce(result, remaining);
2834
+ }
2835
+ // A sweep proves quiescence only if it could have seen everything this NODE issued, not merely what
2836
+ // this coordinator holds. Two ways it could not have:
2837
+ //
2838
+ // - No coordinator exists for the database on this thread, so there is nothing to attest from.
2839
+ // - A coordinator has not owned coordination long enough for authority issued BEFORE it took over
2840
+ // — by a previous owner thread, or a previous incarnation of this process — to have expired.
2841
+ // Those grants live on delegate nodes and no local sweep can see them; the same fact is what
2842
+ // core's own grant gate refuses on, and it is why uptime is not the right measure (a worker that
2843
+ // built empty coordinators early and took ownership late has plenty of uptime and no proof).
2844
+ if (coordinators.length === 0) {
2845
+ result.outstanding.push({
2846
+ table: '*',
2847
+ key: undefined,
2848
+ reason:
2849
+ 'no lock coordinator exists for this database on this thread, so there is nothing to prove quiescence from',
2850
+ });
2851
+ }
2852
+ for (const coordinator of coordinators) {
2853
+ const unproven = coordinator.unprovenOwnershipMs();
2854
+ if (unproven > 0)
2855
+ result.outstanding.push({
2856
+ table: coordinator.table,
2857
+ key: undefined,
2858
+ reason: `this thread has not coordinated ${database}.${coordinator.table} long enough to rule out authority issued before it took over; ${Math.ceil(unproven)}ms remain`,
2859
+ });
2860
+ }
2861
+ // A coordinator that closed parked the latest deadline of the grants it had issued to OTHER nodes
2862
+ // here and then cleared its own table (`close`). Those grants are still valid on their delegates and
2863
+ // no live coordinator holds them, so a sweep that ignored this would miss them entirely — the table
2864
+ // may not even have a coordinator any more.
2865
+ //
2866
+ // `performance.now()` because a retired entry outlives the coordinator whose injected clock produced
2867
+ // its deadline: production passes that same clock (the transport's `monotonicNow`), so the domains
2868
+ // agree where it matters, and a test on an artificial clock only ever reads the deadline as further
2869
+ // away — conservative, never a false clean.
2870
+ const now = performance.now();
2871
+ for (const [key, retired] of retiredCoordinators) {
2872
+ const separator = key.indexOf('\u0000');
2873
+ if (separator < 0 || key.slice(0, separator) !== database) continue;
2874
+ if (!(retired.grantableAfterMono > now)) continue;
2875
+ result.outstanding.push({
2876
+ table: key.slice(separator + 1),
2877
+ key: undefined,
2878
+ reason: `a closed coordinator for this table issued grants that remain valid on their delegates for another ${Math.ceil(retired.grantableAfterMono - now)}ms`,
2879
+ });
2880
+ }
2881
+ if (result.outstanding.length > 0) result.complete = false;
2882
+ return result;
2883
+ }
2884
+
2885
+ /**
2886
+ * Commit a `lockBarrier` entry for the table and resolve to its transaction-log position — the §7.2
2887
+ * recovery fence, for a transport answering a peer's recovery probe. The entry is appended after
2888
+ * every transaction this node had committed when the call was made, so a peer that has applied this
2889
+ * origin's log through the returned position has applied all of them.
2890
+ *
2891
+ * The transport supplies the nonce it will match the entry on, since a position alone is not an
2892
+ * identity: a restart after the wall clock moved backwards can reissue a log key, and a drain that
2893
+ * matched the earlier entry at that key would declare this origin drained with its post-restart
2894
+ * commits unapplied.
2895
+ *
2896
+ * Strictly this node's own commit, never the transport's `writeControl`: the fence is a position in
2897
+ * THIS origin's log, and the caller is the transport itself — a relaying hook would answer with a
2898
+ * position that is not local, or re-enter the operation that called here. A write that commits
2899
+ * without a position rejects rather than resolve, since a barrier nobody can wait on is not a fence.
2900
+ */
2901
+ export async function writeLockBarrier(database: string, table: string, nonce: number): Promise<number> {
2902
+ if (!Number.isSafeInteger(nonce) || nonce < 0)
2903
+ throw new ClientError('A lock barrier nonce must be a non-negative integer');
2904
+ const write = controlWriterResolver?.(database, table);
2905
+ if (!write) throw new ClientError(`Table ${database}.${table} does not exist`, 404);
2906
+ const position = await write({ type: 'lockBarrier', nonce });
2907
+ if (typeof position !== 'number' || !(position >= 0) || !Number.isFinite(position))
2908
+ throw new LockUnavailableError(`the record lock barrier for ${database}.${table} committed without a log position`);
2909
+ return position;
2910
+ }
2911
+
2912
+ /**
2913
+ * Register harper-pro's transport for a database, on THIS thread.
2914
+ *
2915
+ * **It must be registered on every worker that can serve a `lock()`, not only the coordinating one**
2916
+ * — including a dedicated application worker (harper#2524) — and core cannot check that. `clusterRequiredDatabases` is module state, so a worker that never registers never latches —
2917
+ * and a default-scoped `lock()` there takes the Phase 0 node lock alone while a peer runs the cluster
2918
+ * protocol, which is two nodes admitting one key. The `ownsCoordination()` fail-closed path only
2919
+ * reaches a worker that has a transport. Registering everywhere also makes that path the one a
2920
+ * non-owner worker takes, which is what it exists for.
2921
+ */
2922
+ export function registerClusterLockTransport(database: string, transport: ClusterLockTransport): void {
2923
+ if (
2924
+ typeof transport?.homeMap !== 'function' ||
2925
+ typeof transport?.ownsCoordination !== 'function' ||
2926
+ typeof transport?.requestDelegation !== 'function' ||
2927
+ typeof transport?.recallDelegation !== 'function' ||
2928
+ typeof transport?.establishLockFreshness !== 'function'
2929
+ )
2930
+ throw new ClientError(
2931
+ 'A cluster lock transport must provide homeMap(), ownsCoordination(), requestDelegation(), recallDelegation() and establishLockFreshness()'
2932
+ );
2933
+ transport.onControlEntry = (db: string, table: string, entry: LockControlEntry, author: string, position: number) =>
2934
+ deliverLockControlEntry(db, table, entry, author, position);
2935
+ transport.onDelegationRequest = (db: string, table: string, request: DelegationRequest) =>
2936
+ deliverDelegationRequest(db, table, request);
2937
+ transport.onDelegationRecall = (db: string, table: string, recall: DelegationRecall) =>
2938
+ deliverDelegationRecall(db, table, recall);
2939
+ clusterRequiredDatabases.add(database);
2940
+ clusterLockTransports.set(database, transport);
2941
+ }
2942
+
2943
+ export function unregisterClusterLockTransport(database: string, standalone = false): void {
2944
+ clusterLockTransports.delete(database);
2945
+ // Only an explicit statement that the database is no longer clustered clears the requirement.
2946
+ if (standalone) clusterRequiredDatabases.delete(database);
2947
+ }
2948
+
2949
+ /** True once a transport has been registered for this database and no standalone claim has cleared it. */
2950
+ export function isClusterLockRequired(database: string): boolean {
2951
+ return clusterRequiredDatabases.size > 0 && clusterRequiredDatabases.has(database);
2952
+ }
2953
+
2954
+ /**
2955
+ * The registered transport, if any. The `size` check keeps the Phase 0 path free of a map lookup on
2956
+ * every `lock()` in a build where no transport is ever registered.
2957
+ */
2958
+ export function getClusterLockTransport(database: string): ClusterLockTransport | undefined {
2959
+ if (clusterLockTransports.size === 0) return undefined;
2960
+ return clusterLockTransports.get(database);
2961
+ }
2962
+
2963
+ export function hasClusterLockTransports(): boolean {
2964
+ return clusterLockTransports.size > 0;
2965
+ }
2966
+
2967
+ /**
2968
+ * Resolve a coordinator for an inbound message. The resolver reaches `Table.lockCoordinator`, which
2969
+ * throws when this node's name is unusable — that throw must not escape a receive boundary, or it
2970
+ * reaches the replicated apply loop and drops the enclosing transaction.
2971
+ */
2972
+ function coordinatorFor(database: string, table: string, resolve = coordinatorResolver): LockCoordinator | undefined {
2973
+ try {
2974
+ return resolve?.(database, table);
2975
+ } catch (error) {
2976
+ warnOnce('could not resolve a record lock coordinator for a received message', error);
2977
+ return undefined;
2978
+ }
2979
+ }
2980
+
2981
+ export function deliverLockControlEntry(
2982
+ database: string,
2983
+ table: string,
2984
+ entry: LockControlEntry,
2985
+ author: string,
2986
+ position: number
2987
+ ): void {
2988
+ coordinatorFor(database, table, admittingResolver)?.applyEntry(entry, author, position);
2989
+ }
2990
+
2991
+ export async function deliverDelegationRequest(
2992
+ database: string,
2993
+ table: string,
2994
+ request: DelegationRequest
2995
+ ): Promise<DelegationReply> {
2996
+ const coordinator = coordinatorFor(database, table);
2997
+ if (!coordinator) return { granted: false, reason: 'not-home' };
2998
+ try {
2999
+ return await coordinator.onDelegationRequest(request);
3000
+ } catch (error) {
3001
+ warnOnce('failed to answer a record lock delegation request', error);
3002
+ return { granted: false, reason: 'not-home' };
3003
+ }
3004
+ }
3005
+
3006
+ export async function deliverDelegationRecall(
3007
+ database: string,
3008
+ table: string,
3009
+ recall: DelegationRecall
3010
+ ): Promise<void> {
3011
+ // This is core's receiving end of `recallDelegation`, which resolves only "once the delegate has
3012
+ // drained and stopped admitting" — and `#beginRecall` latches `recallConfirmed` on that resolution
3013
+ // and never re-sends. So a recall this thread could not apply has to FAIL rather than resolve:
3014
+ // `coordinatorFor` is the transport-gated resolver and answers undefined through a reconnect, and
3015
+ // the home reading that silence as a drained delegate denies the key to every other node for the
3016
+ // delegation's whole deadline. Failing it leaves the home's `.catch` to retry on RECALL_RETRY_MS.
3017
+ const coordinator = coordinatorFor(database, table);
3018
+ if (!coordinator)
3019
+ throw new Error(`No record lock coordinator on this thread to apply a recall for ${database}.${table}`);
3020
+ await coordinator.onDelegationRecall(recall);
3021
+ }
3022
+
3023
+ // ---- owner-worker relay (harper-pro#852) -------------------------------------------------------
3024
+
3025
+ /**
3026
+ * The owner thread's end of `acquireOnOwner`: mint an admission for a `lock()` served on another
3027
+ * worker thread, and register `revoke` as the way to fence that thread's handle. `revoke` is what the
3028
+ * transport wires to a cross-thread message; a recall or surrender here calls it and waits for the
3029
+ * fence before writing the release. Runs on the coordinating thread, resolved through the transport-
3030
+ * gated resolver so it fails when this thread does not coordinate the database — the same shape as
3031
+ * `deliverDelegationRequest`. Returns the round the calling worker installs as a remote admission.
3032
+ */
3033
+ export async function acquireForRelay(
3034
+ database: string,
3035
+ table: string,
3036
+ key: any,
3037
+ leaseMs: number,
3038
+ waitMs: number,
3039
+ makeRevoke: (round: LockRound) => () => void | Promise<void>
3040
+ ): Promise<LockRound> {
3041
+ const coordinator = coordinatorFor(database, table);
3042
+ if (!coordinator)
3043
+ throw new Error(`No record lock coordinator on this thread to acquire ${database}.${table} for a peer worker`);
3044
+ const round = await coordinator.acquire(key, leaseMs, waitMs);
3045
+ // Re-resolve through the ADMITTING resolver rather than reusing the captured coordinator: a transport
3046
+ // swap during the acquire moves the admission to the successor and empties the predecessor, so
3047
+ // registering on the captured object would eagerly revoke a healthy handle and leave the successor's
3048
+ // admission unfenceable. `Table.ts` does the same via `admittingCoordinator`. The revoker is built
3049
+ // from the round so it names the exact admission when it tells the calling worker to fence its handle.
3050
+ const authority = coordinatorFor(database, table, admittingResolver) ?? coordinator;
3051
+ authority.registerAdmission(round.admissionId, makeRevoke(round));
3052
+ return round;
3053
+ }
3054
+
3055
+ /** The owner thread's end of `releaseOnOwner`: end a relayed admission the owner minted. */
3056
+ export function releaseForRelay(database: string, table: string, key: any, admissionId: number): Promise<void> | void {
3057
+ // The admitting resolver, like a received release: it answers the coordinator that HOLDS the
3058
+ // admission even while a transport is momentarily unregistered, so a release is never dropped.
3059
+ return coordinatorFor(database, table, admittingResolver)?.release(key, admissionId);
3060
+ }
3061
+
3062
+ /**
3063
+ * The calling thread's end of an owner `revoke`: fence the handle for a relayed admission this thread
3064
+ * holds. Resolves once the handle's `revokeLease` has run, so the owner may wait for the fence before
3065
+ * it writes the release. The admitting resolver answers the coordinator that adopted the admission
3066
+ * across a transport swap.
3067
+ */
3068
+ export function revokeRelayedAdmission(database: string, table: string, admissionId: number): Promise<void> {
3069
+ const coordinator = coordinatorFor(database, table, admittingResolver);
3070
+ // No coordinator to fence against means nothing here can commit under that admission; the fence is
3071
+ // vacuously satisfied and the owner may proceed.
3072
+ return coordinator ? coordinator.revokeRemoteAdmission(admissionId) : Promise.resolve();
3073
+ }
3074
+
3075
+ /**
3076
+ * Fail-closed fence for every relayed handle a table's coordinator holds (harper-pro#852), for when
3077
+ * the owner worker that granted them has exited and its delegations are gone. Harper-pro calls this
3078
+ * per table when it learns the coordinating thread for a database changed.
3079
+ */
3080
+ export function fenceRelayedAdmissions(database: string, table: string): void {
3081
+ coordinatorFor(database, table, admittingResolver)?.fenceAllRemoteAdmissions();
3082
+ }