@harperfast/harper 5.2.13 → 5.3.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (655) hide show
  1. package/agent/mcpTools.ts +1 -1
  2. package/agent/session.ts +25 -14
  3. package/bin/cliOperations.ts +58 -9
  4. package/bin/copyDb.ts +282 -59
  5. package/bin/deploySetup.ts +16 -5
  6. package/bin/harper.ts +1 -1
  7. package/bin/help.ts +4 -1
  8. package/bin/lite.ts +4 -1
  9. package/bin/restart.ts +120 -4
  10. package/bin/run.ts +6 -11
  11. package/bin/upgrade.js +7 -3
  12. package/bin/workloadIdentity.ts +119 -0
  13. package/components/Application.ts +3436 -242
  14. package/components/ApplicationScope.ts +8 -0
  15. package/components/EntryHandler.ts +59 -39
  16. package/components/OptionsWatcher.ts +440 -98
  17. package/components/RuntimeModuleTracker.ts +38 -7
  18. package/components/Scope.ts +56 -15
  19. package/components/awaitRestart.ts +84 -0
  20. package/components/componentLoader.ts +381 -32
  21. package/components/componentPreparationLock.ts +16 -5
  22. package/components/deploymentOperations.ts +4 -1
  23. package/components/deploymentRecorder.ts +9 -2
  24. package/components/mcp/adapters/harperHttp.ts +4 -0
  25. package/components/mcp/listChanged.ts +4 -0
  26. package/components/mcp/toolRegistry.ts +2 -0
  27. package/components/mcp/tools/operations.ts +9 -0
  28. package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
  29. package/components/operations.js +537 -113
  30. package/components/operationsValidation.js +98 -3
  31. package/components/packageComponent.ts +25 -1
  32. package/components/requestRestart.ts +11 -0
  33. package/components/status/ComponentStatusRegistry.ts +59 -0
  34. package/config/RootConfigWatcher.ts +240 -40
  35. package/config/configReadRetry.ts +62 -0
  36. package/config/configUtils.ts +357 -48
  37. package/config/harperConfigEnvVars.ts +170 -27
  38. package/config/parseConfigFile.ts +34 -0
  39. package/config/readConfigFileSync.ts +44 -0
  40. package/config/watcherArming.ts +59 -0
  41. package/config-root.schema.json +33 -0
  42. package/dataLayer/blobBackup.ts +160 -50
  43. package/dataLayer/delete.ts +6 -1
  44. package/dataLayer/harperBridge/ResourceBridge.ts +80 -10
  45. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  46. package/dataLayer/hdbInfoController.ts +34 -1
  47. package/dataLayer/insert.ts +44 -1
  48. package/dataLayer/rocksdbBackup.ts +53 -10
  49. package/dataLayer/schema.ts +11 -1
  50. package/dataLayer/schemaDescribe.ts +8 -1
  51. package/dist/agent/mcpTools.js +1 -1
  52. package/dist/agent/mcpTools.js.map +1 -1
  53. package/dist/agent/session.d.ts +22 -0
  54. package/dist/agent/session.js +26 -15
  55. package/dist/agent/session.js.map +1 -1
  56. package/dist/bin/cliOperations.js +61 -9
  57. package/dist/bin/cliOperations.js.map +1 -1
  58. package/dist/bin/copyDb.d.ts +12 -1
  59. package/dist/bin/copyDb.js +248 -60
  60. package/dist/bin/copyDb.js.map +1 -1
  61. package/dist/bin/deploySetup.d.ts +2 -0
  62. package/dist/bin/deploySetup.js +11 -3
  63. package/dist/bin/deploySetup.js.map +1 -1
  64. package/dist/bin/harper.js +1 -1
  65. package/dist/bin/harper.js.map +1 -1
  66. package/dist/bin/help.js +4 -1
  67. package/dist/bin/help.js.map +1 -1
  68. package/dist/bin/lite.js +4 -1
  69. package/dist/bin/lite.js.map +1 -1
  70. package/dist/bin/restart.js +91 -6
  71. package/dist/bin/restart.js.map +1 -1
  72. package/dist/bin/run.js +4 -10
  73. package/dist/bin/run.js.map +1 -1
  74. package/dist/bin/upgrade.js +4 -3
  75. package/dist/bin/upgrade.js.map +1 -1
  76. package/dist/bin/workloadIdentity.d.ts +18 -0
  77. package/dist/bin/workloadIdentity.js +100 -0
  78. package/dist/bin/workloadIdentity.js.map +1 -0
  79. package/dist/components/Application.d.ts +235 -17
  80. package/dist/components/Application.js +3038 -238
  81. package/dist/components/Application.js.map +1 -1
  82. package/dist/components/ApplicationScope.d.ts +8 -0
  83. package/dist/components/ApplicationScope.js +7 -0
  84. package/dist/components/ApplicationScope.js.map +1 -1
  85. package/dist/components/EntryHandler.js +26 -10
  86. package/dist/components/EntryHandler.js.map +1 -1
  87. package/dist/components/OptionsWatcher.d.ts +4 -0
  88. package/dist/components/OptionsWatcher.js +440 -99
  89. package/dist/components/OptionsWatcher.js.map +1 -1
  90. package/dist/components/RuntimeModuleTracker.js +40 -6
  91. package/dist/components/RuntimeModuleTracker.js.map +1 -1
  92. package/dist/components/Scope.js +52 -13
  93. package/dist/components/Scope.js.map +1 -1
  94. package/dist/components/awaitRestart.d.ts +33 -0
  95. package/dist/components/awaitRestart.js +61 -0
  96. package/dist/components/awaitRestart.js.map +1 -0
  97. package/dist/components/componentLoader.d.ts +38 -1
  98. package/dist/components/componentLoader.js +313 -24
  99. package/dist/components/componentLoader.js.map +1 -1
  100. package/dist/components/componentPreparationLock.d.ts +5 -0
  101. package/dist/components/componentPreparationLock.js +14 -6
  102. package/dist/components/componentPreparationLock.js.map +1 -1
  103. package/dist/components/deploymentOperations.js +4 -1
  104. package/dist/components/deploymentOperations.js.map +1 -1
  105. package/dist/components/deploymentRecorder.d.ts +4 -2
  106. package/dist/components/deploymentRecorder.js +1 -0
  107. package/dist/components/deploymentRecorder.js.map +1 -1
  108. package/dist/components/mcp/adapters/harperHttp.js +4 -0
  109. package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
  110. package/dist/components/mcp/listChanged.js +5 -0
  111. package/dist/components/mcp/listChanged.js.map +1 -1
  112. package/dist/components/mcp/toolRegistry.d.ts +1 -0
  113. package/dist/components/mcp/toolRegistry.js.map +1 -1
  114. package/dist/components/mcp/tools/operations.d.ts +5 -0
  115. package/dist/components/mcp/tools/operations.js +9 -0
  116. package/dist/components/mcp/tools/operations.js.map +1 -1
  117. package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
  118. package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
  119. package/dist/components/operations.d.ts +28 -0
  120. package/dist/components/operations.js +476 -113
  121. package/dist/components/operations.js.map +1 -1
  122. package/dist/components/operationsValidation.js +97 -3
  123. package/dist/components/operationsValidation.js.map +1 -1
  124. package/dist/components/packageComponent.js +24 -0
  125. package/dist/components/packageComponent.js.map +1 -1
  126. package/dist/components/requestRestart.d.ts +1 -0
  127. package/dist/components/requestRestart.js +7 -0
  128. package/dist/components/requestRestart.js.map +1 -1
  129. package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
  130. package/dist/components/status/ComponentStatusRegistry.js +63 -0
  131. package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
  132. package/dist/config/RootConfigWatcher.d.ts +9 -1
  133. package/dist/config/RootConfigWatcher.js +228 -37
  134. package/dist/config/RootConfigWatcher.js.map +1 -1
  135. package/dist/config/configReadRetry.d.ts +8 -0
  136. package/dist/config/configReadRetry.js +62 -0
  137. package/dist/config/configReadRetry.js.map +1 -0
  138. package/dist/config/configUtils.d.ts +11 -2
  139. package/dist/config/configUtils.js +304 -47
  140. package/dist/config/configUtils.js.map +1 -1
  141. package/dist/config/harperConfigEnvVars.d.ts +16 -0
  142. package/dist/config/harperConfigEnvVars.js +162 -25
  143. package/dist/config/harperConfigEnvVars.js.map +1 -1
  144. package/dist/config/parseConfigFile.d.ts +4 -0
  145. package/dist/config/parseConfigFile.js +35 -0
  146. package/dist/config/parseConfigFile.js.map +1 -0
  147. package/dist/config/readConfigFileSync.d.ts +1 -0
  148. package/dist/config/readConfigFileSync.js +47 -0
  149. package/dist/config/readConfigFileSync.js.map +1 -0
  150. package/dist/config/watcherArming.d.ts +15 -0
  151. package/dist/config/watcherArming.js +59 -0
  152. package/dist/config/watcherArming.js.map +1 -0
  153. package/dist/dataLayer/blobBackup.d.ts +49 -20
  154. package/dist/dataLayer/blobBackup.js +139 -50
  155. package/dist/dataLayer/blobBackup.js.map +1 -1
  156. package/dist/dataLayer/delete.js +1 -1
  157. package/dist/dataLayer/delete.js.map +1 -1
  158. package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
  159. package/dist/dataLayer/harperBridge/ResourceBridge.js +72 -12
  160. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  161. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
  162. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  163. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
  164. package/dist/dataLayer/hdbInfoController.d.ts +10 -0
  165. package/dist/dataLayer/hdbInfoController.js +30 -1
  166. package/dist/dataLayer/hdbInfoController.js.map +1 -1
  167. package/dist/dataLayer/insert.d.ts +9 -1
  168. package/dist/dataLayer/insert.js +30 -0
  169. package/dist/dataLayer/insert.js.map +1 -1
  170. package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
  171. package/dist/dataLayer/rocksdbBackup.js +45 -8
  172. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  173. package/dist/dataLayer/schema.js +8 -0
  174. package/dist/dataLayer/schema.js.map +1 -1
  175. package/dist/dataLayer/schemaDescribe.js +8 -1
  176. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  177. package/dist/index.d.ts +2 -1
  178. package/dist/index.js +4 -1
  179. package/dist/index.js.map +1 -1
  180. package/dist/json/systemSchema.json +55 -0
  181. package/dist/resources/DatabaseTransaction.d.ts +77 -0
  182. package/dist/resources/DatabaseTransaction.js +586 -54
  183. package/dist/resources/DatabaseTransaction.js.map +1 -1
  184. package/dist/resources/LMDBTransaction.d.ts +2 -1
  185. package/dist/resources/LMDBTransaction.js +43 -6
  186. package/dist/resources/LMDBTransaction.js.map +1 -1
  187. package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
  188. package/dist/resources/PrimaryRocksDatabase.js +49 -5
  189. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  190. package/dist/resources/RecordEncoder.d.ts +20 -0
  191. package/dist/resources/RecordEncoder.js +110 -9
  192. package/dist/resources/RecordEncoder.js.map +1 -1
  193. package/dist/resources/RequestTarget.d.ts +2 -0
  194. package/dist/resources/RequestTarget.js.map +1 -1
  195. package/dist/resources/Resource.js +117 -24
  196. package/dist/resources/Resource.js.map +1 -1
  197. package/dist/resources/ResourceInterface.d.ts +28 -1
  198. package/dist/resources/ResourceInterface.js.map +1 -1
  199. package/dist/resources/RocksIndexStore.d.ts +6 -1
  200. package/dist/resources/RocksIndexStore.js +24 -9
  201. package/dist/resources/RocksIndexStore.js.map +1 -1
  202. package/dist/resources/RocksTransactionLogStore.d.ts +24 -1
  203. package/dist/resources/RocksTransactionLogStore.js +160 -49
  204. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  205. package/dist/resources/Table.d.ts +165 -10
  206. package/dist/resources/Table.js +1805 -303
  207. package/dist/resources/Table.js.map +1 -1
  208. package/dist/resources/analytics/write.js +10 -3
  209. package/dist/resources/analytics/write.js.map +1 -1
  210. package/dist/resources/auditStore.d.ts +181 -2
  211. package/dist/resources/auditStore.js +640 -29
  212. package/dist/resources/auditStore.js.map +1 -1
  213. package/dist/resources/blob.d.ts +129 -9
  214. package/dist/resources/blob.js +992 -114
  215. package/dist/resources/blob.js.map +1 -1
  216. package/dist/resources/branchDatabase.d.ts +48 -0
  217. package/dist/resources/branchDatabase.js +892 -0
  218. package/dist/resources/branchDatabase.js.map +1 -0
  219. package/dist/resources/crdt.js +50 -12
  220. package/dist/resources/crdt.js.map +1 -1
  221. package/dist/resources/dataLoader.js +3 -4
  222. package/dist/resources/dataLoader.js.map +1 -1
  223. package/dist/resources/databases.d.ts +157 -14
  224. package/dist/resources/databases.js +1462 -325
  225. package/dist/resources/databases.js.map +1 -1
  226. package/dist/resources/defineTable.d.ts +10 -2
  227. package/dist/resources/defineTable.js +9 -1
  228. package/dist/resources/defineTable.js.map +1 -1
  229. package/dist/resources/derivedIndexRegistry.d.ts +5 -0
  230. package/dist/resources/derivedIndexRegistry.js +68 -0
  231. package/dist/resources/derivedIndexRegistry.js.map +1 -0
  232. package/dist/resources/derivedIndexRuntime.d.ts +215 -0
  233. package/dist/resources/derivedIndexRuntime.js +2027 -0
  234. package/dist/resources/derivedIndexRuntime.js.map +1 -0
  235. package/dist/resources/graphql.d.ts +1 -1
  236. package/dist/resources/graphql.js +52 -16
  237. package/dist/resources/graphql.js.map +1 -1
  238. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +103 -9
  239. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +854 -41
  240. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  241. package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
  242. package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
  243. package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
  244. package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
  245. package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
  246. package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
  247. package/dist/resources/longLivedTransactions.d.ts +71 -0
  248. package/dist/resources/longLivedTransactions.js +358 -0
  249. package/dist/resources/longLivedTransactions.js.map +1 -0
  250. package/dist/resources/models/backendRegistry.d.ts +26 -0
  251. package/dist/resources/models/backendRegistry.js +60 -2
  252. package/dist/resources/models/backendRegistry.js.map +1 -1
  253. package/dist/resources/models/bootstrap.d.ts +33 -1
  254. package/dist/resources/models/bootstrap.js +416 -31
  255. package/dist/resources/models/bootstrap.js.map +1 -1
  256. package/dist/resources/nodeIdMapping.d.ts +5 -0
  257. package/dist/resources/nodeIdMapping.js +49 -0
  258. package/dist/resources/nodeIdMapping.js.map +1 -1
  259. package/dist/resources/recordLock.d.ts +123 -0
  260. package/dist/resources/recordLock.js +315 -0
  261. package/dist/resources/recordLock.js.map +1 -0
  262. package/dist/resources/recordLockCoordinator.d.ts +557 -0
  263. package/dist/resources/recordLockCoordinator.js +2565 -0
  264. package/dist/resources/recordLockCoordinator.js.map +1 -0
  265. package/dist/resources/replayLogs.d.ts +14 -1
  266. package/dist/resources/replayLogs.js +157 -24
  267. package/dist/resources/replayLogs.js.map +1 -1
  268. package/dist/resources/replayLogsGuards.d.ts +94 -7
  269. package/dist/resources/replayLogsGuards.js +111 -7
  270. package/dist/resources/replayLogsGuards.js.map +1 -1
  271. package/dist/resources/replicatedApplyFailure.d.ts +16 -0
  272. package/dist/resources/replicatedApplyFailure.js +63 -0
  273. package/dist/resources/replicatedApplyFailure.js.map +1 -0
  274. package/dist/resources/scheduler/scheduler.js +3 -3
  275. package/dist/resources/scheduler/scheduler.js.map +1 -1
  276. package/dist/resources/search.d.ts +10 -4
  277. package/dist/resources/search.js +283 -37
  278. package/dist/resources/search.js.map +1 -1
  279. package/dist/resources/tracked.d.ts +5 -1
  280. package/dist/resources/tracked.js +74 -23
  281. package/dist/resources/tracked.js.map +1 -1
  282. package/dist/resources/transactionBroadcast.d.ts +1 -1
  283. package/dist/resources/transactionBroadcast.js +2 -2
  284. package/dist/resources/transactionBroadcast.js.map +1 -1
  285. package/dist/security/auth.js +61 -30
  286. package/dist/security/auth.js.map +1 -1
  287. package/dist/security/authn/oidc/claims.d.ts +22 -0
  288. package/dist/security/authn/oidc/claims.js +71 -0
  289. package/dist/security/authn/oidc/claims.js.map +1 -0
  290. package/dist/security/authn/oidc/identityToken.d.ts +27 -0
  291. package/dist/security/authn/oidc/identityToken.js +111 -0
  292. package/dist/security/authn/oidc/identityToken.js.map +1 -0
  293. package/dist/security/authn/oidc/jwks.d.ts +25 -0
  294. package/dist/security/authn/oidc/jwks.js +261 -0
  295. package/dist/security/authn/oidc/jwks.js.map +1 -0
  296. package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
  297. package/dist/security/authn/oidc/providers/generic.js +34 -0
  298. package/dist/security/authn/oidc/providers/generic.js.map +1 -0
  299. package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
  300. package/dist/security/authn/oidc/providers/githubActions.js +129 -0
  301. package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
  302. package/dist/security/authn/oidc/providers/index.d.ts +37 -0
  303. package/dist/security/authn/oidc/providers/index.js +24 -0
  304. package/dist/security/authn/oidc/providers/index.js.map +1 -0
  305. package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
  306. package/dist/security/authn/oidc/tokenExchange.js +306 -0
  307. package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
  308. package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
  309. package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
  310. package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
  311. package/dist/security/authn/oidc/types.d.ts +38 -0
  312. package/dist/security/authn/oidc/types.js +6 -0
  313. package/dist/security/authn/oidc/types.js.map +1 -0
  314. package/dist/security/certificateVerification/index.js +40 -11
  315. package/dist/security/certificateVerification/index.js.map +1 -1
  316. package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
  317. package/dist/security/certificateVerification/trustedIssuers.js +79 -0
  318. package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
  319. package/dist/security/certificateVerification/types.d.ts +1 -0
  320. package/dist/security/credentialProvenance.d.ts +35 -0
  321. package/dist/security/credentialProvenance.js +51 -0
  322. package/dist/security/credentialProvenance.js.map +1 -0
  323. package/dist/security/credentialRejection.d.ts +4 -0
  324. package/dist/security/credentialRejection.js +24 -0
  325. package/dist/security/credentialRejection.js.map +1 -0
  326. package/dist/security/deferredAuthentication.d.ts +36 -0
  327. package/dist/security/deferredAuthentication.js +70 -0
  328. package/dist/security/deferredAuthentication.js.map +1 -0
  329. package/dist/security/impersonation.d.ts +21 -0
  330. package/dist/security/impersonation.js +108 -9
  331. package/dist/security/impersonation.js.map +1 -1
  332. package/dist/security/jsLoader.d.ts +6 -0
  333. package/dist/security/jsLoader.js +77 -15
  334. package/dist/security/jsLoader.js.map +1 -1
  335. package/dist/security/keys.js +301 -71
  336. package/dist/security/keys.js.map +1 -1
  337. package/dist/security/operationScope.d.ts +21 -0
  338. package/dist/security/operationScope.js +36 -0
  339. package/dist/security/operationScope.js.map +1 -0
  340. package/dist/security/permissionsTranslator.js +21 -0
  341. package/dist/security/permissionsTranslator.js.map +1 -1
  342. package/dist/security/tokenAuthentication.d.ts +19 -1
  343. package/dist/security/tokenAuthentication.js +191 -10
  344. package/dist/security/tokenAuthentication.js.map +1 -1
  345. package/dist/security/user.js +4 -3
  346. package/dist/security/user.js.map +1 -1
  347. package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
  348. package/dist/server/DurableSubscriptionsSession.js +67 -10
  349. package/dist/server/DurableSubscriptionsSession.js.map +1 -1
  350. package/dist/server/REST.js +106 -2
  351. package/dist/server/REST.js.map +1 -1
  352. package/dist/server/graphqlQuerying.js +4 -0
  353. package/dist/server/graphqlQuerying.js.map +1 -1
  354. package/dist/server/http.d.ts +16 -1
  355. package/dist/server/http.js +122 -17
  356. package/dist/server/http.js.map +1 -1
  357. package/dist/server/itc/serverHandlers.js +8 -1
  358. package/dist/server/itc/serverHandlers.js.map +1 -1
  359. package/dist/server/jobs/jobProcess.js +6 -2
  360. package/dist/server/jobs/jobProcess.js.map +1 -1
  361. package/dist/server/jobs/jobs.js +4 -1
  362. package/dist/server/jobs/jobs.js.map +1 -1
  363. package/dist/server/liveSubscriptionAuth.d.ts +26 -4
  364. package/dist/server/liveSubscriptionAuth.js +105 -39
  365. package/dist/server/liveSubscriptionAuth.js.map +1 -1
  366. package/dist/server/loadRootComponents.js +49 -10
  367. package/dist/server/loadRootComponents.js.map +1 -1
  368. package/dist/server/mqtt.d.ts +2 -0
  369. package/dist/server/mqtt.js +165 -30
  370. package/dist/server/mqtt.js.map +1 -1
  371. package/dist/server/nodeName.d.ts +2 -0
  372. package/dist/server/nodeName.js +107 -23
  373. package/dist/server/nodeName.js.map +1 -1
  374. package/dist/server/serverHelpers/Headers.d.ts +27 -0
  375. package/dist/server/serverHelpers/Headers.js +145 -1
  376. package/dist/server/serverHelpers/Headers.js.map +1 -1
  377. package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
  378. package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
  379. package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
  380. package/dist/server/serverHelpers/Request.d.ts +5 -10
  381. package/dist/server/serverHelpers/Request.js +38 -136
  382. package/dist/server/serverHelpers/Request.js.map +1 -1
  383. package/dist/server/serverHelpers/contentTypes.d.ts +11 -0
  384. package/dist/server/serverHelpers/contentTypes.js +221 -40
  385. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  386. package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
  387. package/dist/server/serverHelpers/deployValidationState.js +9 -19
  388. package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
  389. package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
  390. package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
  391. package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
  392. package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
  393. package/dist/server/serverHelpers/registeredOperations.js +74 -21
  394. package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
  395. package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
  396. package/dist/server/serverHelpers/requestSanitization.js +20 -0
  397. package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
  398. package/dist/server/serverHelpers/serverHandlers.js +6 -3
  399. package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
  400. package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
  401. package/dist/server/serverHelpers/serverUtilities.js +178 -29
  402. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  403. package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
  404. package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
  405. package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
  406. package/dist/server/serverHelpers/uwsServer.js +19 -1
  407. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  408. package/dist/server/static.js +24 -28
  409. package/dist/server/static.js.map +1 -1
  410. package/dist/server/storageReclamation.d.ts +5 -0
  411. package/dist/server/storageReclamation.js +17 -1
  412. package/dist/server/storageReclamation.js.map +1 -1
  413. package/dist/server/threads/isolatedApplications.d.ts +47 -0
  414. package/dist/server/threads/isolatedApplications.js +171 -0
  415. package/dist/server/threads/isolatedApplications.js.map +1 -0
  416. package/dist/server/threads/itc.d.ts +7 -2
  417. package/dist/server/threads/itc.js +5 -1
  418. package/dist/server/threads/itc.js.map +1 -1
  419. package/dist/server/threads/logRotationTransport.d.ts +1 -0
  420. package/dist/server/threads/logRotationTransport.js +33 -0
  421. package/dist/server/threads/logRotationTransport.js.map +1 -0
  422. package/dist/server/threads/manageThreads.d.ts +83 -3
  423. package/dist/server/threads/manageThreads.js +769 -75
  424. package/dist/server/threads/manageThreads.js.map +1 -1
  425. package/dist/server/threads/socketRouter.d.ts +1 -0
  426. package/dist/server/threads/socketRouter.js +277 -31
  427. package/dist/server/threads/socketRouter.js.map +1 -1
  428. package/dist/server/threads/threadHeapMemory.d.ts +2 -0
  429. package/dist/server/threads/threadHeapMemory.js +31 -0
  430. package/dist/server/threads/threadHeapMemory.js.map +1 -0
  431. package/dist/server/threads/threadServer.js +76 -22
  432. package/dist/server/threads/threadServer.js.map +1 -1
  433. package/dist/sqlEngine/config.d.ts +1 -3
  434. package/dist/sqlEngine/config.js +19 -16
  435. package/dist/sqlEngine/config.js.map +1 -1
  436. package/dist/sqlTranslator/index.d.ts +1 -1
  437. package/dist/sqlTranslator/index.js +30 -7
  438. package/dist/sqlTranslator/index.js.map +1 -1
  439. package/dist/upgrade/directives/5-3-0.d.ts +7 -0
  440. package/dist/upgrade/directives/5-3-0.js +148 -0
  441. package/dist/upgrade/directives/5-3-0.js.map +1 -0
  442. package/dist/upgrade/directives/directivesController.js +2 -1
  443. package/dist/upgrade/directives/directivesController.js.map +1 -1
  444. package/dist/utility/OperationFunctionCaller.js +2 -1
  445. package/dist/utility/OperationFunctionCaller.js.map +1 -1
  446. package/dist/utility/common_utils.d.ts +16 -0
  447. package/dist/utility/common_utils.js +32 -6
  448. package/dist/utility/common_utils.js.map +1 -1
  449. package/dist/utility/componentNames.d.ts +8 -0
  450. package/dist/utility/componentNames.js +12 -1
  451. package/dist/utility/componentNames.js.map +1 -1
  452. package/dist/utility/environment/environmentManager.js +3 -6
  453. package/dist/utility/environment/environmentManager.js.map +1 -1
  454. package/dist/utility/environment/systemInformation.d.ts +1 -0
  455. package/dist/utility/environment/systemInformation.js +1 -0
  456. package/dist/utility/environment/systemInformation.js.map +1 -1
  457. package/dist/utility/errors/commonErrors.d.ts +2 -0
  458. package/dist/utility/errors/commonErrors.js +2 -0
  459. package/dist/utility/errors/commonErrors.js.map +1 -1
  460. package/dist/utility/errors/hdbError.d.ts +33 -0
  461. package/dist/utility/errors/hdbError.js +58 -1
  462. package/dist/utility/errors/hdbError.js.map +1 -1
  463. package/dist/utility/globalSchema.d.ts +18 -0
  464. package/dist/utility/hdbTerms.d.ts +18 -0
  465. package/dist/utility/hdbTerms.js +20 -2
  466. package/dist/utility/hdbTerms.js.map +1 -1
  467. package/dist/utility/logging/harper_logger.d.ts +2 -0
  468. package/dist/utility/logging/harper_logger.js +286 -30
  469. package/dist/utility/logging/harper_logger.js.map +1 -1
  470. package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
  471. package/dist/utility/logging/logGenerationCoordinator.js +184 -0
  472. package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
  473. package/dist/utility/logging/logRotation.d.ts +46 -0
  474. package/dist/utility/logging/logRotation.js +365 -0
  475. package/dist/utility/logging/logRotation.js.map +1 -0
  476. package/dist/utility/logging/logRotator.d.ts +1 -1
  477. package/dist/utility/logging/logRotator.js +192 -85
  478. package/dist/utility/logging/logRotator.js.map +1 -1
  479. package/dist/utility/nodeIdentity.d.ts +9 -0
  480. package/dist/utility/nodeIdentity.js +58 -0
  481. package/dist/utility/nodeIdentity.js.map +1 -0
  482. package/dist/utility/npmUtilities.js +11 -7
  483. package/dist/utility/npmUtilities.js.map +1 -1
  484. package/dist/utility/operationPermissions.d.ts +3 -1
  485. package/dist/utility/operationPermissions.js +16 -1
  486. package/dist/utility/operationPermissions.js.map +1 -1
  487. package/dist/utility/operation_authorization.d.ts +10 -7
  488. package/dist/utility/operation_authorization.js +212 -41
  489. package/dist/utility/operation_authorization.js.map +1 -1
  490. package/dist/utility/watchPath.d.ts +29 -0
  491. package/dist/utility/watchPath.js +68 -0
  492. package/dist/utility/watchPath.js.map +1 -0
  493. package/dist/utility/watcherFallback.d.ts +41 -0
  494. package/dist/utility/watcherFallback.js +153 -0
  495. package/dist/utility/watcherFallback.js.map +1 -1
  496. package/dist/validation/configValidator.d.ts +12 -0
  497. package/dist/validation/configValidator.js +205 -75
  498. package/dist/validation/configValidator.js.map +1 -1
  499. package/dist/validation/installValidator.js +12 -0
  500. package/dist/validation/installValidator.js.map +1 -1
  501. package/dist/validation/validationWrapper.d.ts +11 -0
  502. package/dist/validation/validationWrapper.js +16 -3
  503. package/dist/validation/validationWrapper.js.map +1 -1
  504. package/index.ts +8 -0
  505. package/json/systemSchema.json +55 -0
  506. package/npm-shrinkwrap.json +286 -194
  507. package/package.json +15 -7
  508. package/resources/DESIGN.md +219 -52
  509. package/resources/DatabaseTransaction.ts +657 -52
  510. package/resources/LMDBTransaction.ts +46 -6
  511. package/resources/PrimaryRocksDatabase.ts +46 -7
  512. package/resources/RecordEncoder.ts +124 -9
  513. package/resources/RequestTarget.ts +2 -0
  514. package/resources/Resource.ts +114 -22
  515. package/resources/ResourceInterface.ts +31 -0
  516. package/resources/RocksIndexStore.ts +30 -9
  517. package/resources/RocksTransactionLogStore.ts +188 -51
  518. package/resources/Table.ts +1960 -301
  519. package/resources/analytics/write.ts +10 -3
  520. package/resources/auditStore.ts +647 -32
  521. package/resources/blob.ts +1029 -110
  522. package/resources/branchDatabase.ts +941 -0
  523. package/resources/crdt.ts +76 -12
  524. package/resources/dataLoader.ts +3 -4
  525. package/resources/databases.ts +1584 -320
  526. package/resources/defineTable.ts +18 -2
  527. package/resources/derivedIndexRegistry.ts +56 -0
  528. package/resources/derivedIndexRuntime.ts +2292 -0
  529. package/resources/graphql.ts +72 -17
  530. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +911 -48
  531. package/resources/indexes/hnswDerivedIndex.ts +531 -0
  532. package/resources/indexes/hnswPlaneBinding.ts +174 -0
  533. package/resources/longLivedTransactions.ts +360 -0
  534. package/resources/models/backendRegistry.ts +84 -2
  535. package/resources/models/bootstrap.ts +473 -28
  536. package/resources/nodeIdMapping.ts +50 -0
  537. package/resources/recordLock.ts +419 -0
  538. package/resources/recordLockCoordinator.ts +3043 -0
  539. package/resources/replayLogs.ts +152 -26
  540. package/resources/replayLogsGuards.ts +171 -8
  541. package/resources/replicatedApplyFailure.ts +77 -0
  542. package/resources/scheduler/scheduler.ts +4 -4
  543. package/resources/search.ts +288 -46
  544. package/resources/tracked.ts +73 -22
  545. package/resources/transactionBroadcast.ts +3 -3
  546. package/security/auth.ts +68 -29
  547. package/security/authn/oidc/claims.ts +72 -0
  548. package/security/authn/oidc/identityToken.ts +129 -0
  549. package/security/authn/oidc/jwks.ts +260 -0
  550. package/security/authn/oidc/providers/generic.ts +40 -0
  551. package/security/authn/oidc/providers/githubActions.ts +137 -0
  552. package/security/authn/oidc/providers/index.ts +52 -0
  553. package/security/authn/oidc/tokenExchange.ts +300 -0
  554. package/security/authn/oidc/trustPolicyOperations.ts +343 -0
  555. package/security/authn/oidc/types.ts +41 -0
  556. package/security/certificateVerification/index.ts +54 -13
  557. package/security/certificateVerification/trustedIssuers.ts +76 -0
  558. package/security/certificateVerification/types.ts +1 -0
  559. package/security/credentialProvenance.ts +47 -0
  560. package/security/credentialRejection.ts +22 -0
  561. package/security/deferredAuthentication.ts +71 -0
  562. package/security/impersonation.ts +117 -12
  563. package/security/jsLoader.ts +83 -18
  564. package/security/keys.ts +298 -72
  565. package/security/operationScope.ts +33 -0
  566. package/security/permissionsTranslator.js +23 -0
  567. package/security/tokenAuthentication.ts +233 -12
  568. package/security/user.ts +4 -3
  569. package/server/DESIGN.md +194 -16
  570. package/server/DurableSubscriptionsSession.ts +71 -11
  571. package/server/REST.ts +115 -4
  572. package/server/graphqlQuerying.ts +4 -0
  573. package/server/http.ts +133 -20
  574. package/server/itc/serverHandlers.js +8 -1
  575. package/server/jobs/jobProcess.ts +8 -2
  576. package/server/jobs/jobs.ts +4 -1
  577. package/server/liveSubscriptionAuth.ts +129 -46
  578. package/server/loadRootComponents.js +50 -8
  579. package/server/mqtt.ts +179 -38
  580. package/server/nodeName.ts +103 -21
  581. package/server/serverHelpers/Headers.ts +135 -0
  582. package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
  583. package/server/serverHelpers/Request.ts +33 -131
  584. package/server/serverHelpers/contentTypes.ts +217 -36
  585. package/server/serverHelpers/deployValidationState.ts +24 -13
  586. package/server/serverHelpers/operationAuthorizationState.ts +34 -3
  587. package/server/serverHelpers/registeredOperations.ts +79 -22
  588. package/server/serverHelpers/requestSanitization.ts +15 -0
  589. package/server/serverHelpers/serverHandlers.js +6 -3
  590. package/server/serverHelpers/serverUtilities.ts +232 -40
  591. package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
  592. package/server/serverHelpers/uwsServer.ts +17 -2
  593. package/server/static.ts +23 -29
  594. package/server/storageReclamation.ts +15 -2
  595. package/server/threads/isolatedApplications.ts +157 -0
  596. package/server/threads/itc.js +11 -1
  597. package/server/threads/logRotationTransport.ts +40 -0
  598. package/server/threads/manageThreads.js +771 -66
  599. package/server/threads/socketRouter.ts +291 -30
  600. package/server/threads/threadHeapMemory.ts +26 -0
  601. package/server/threads/threadServer.js +73 -22
  602. package/sqlTranslator/index.ts +31 -8
  603. package/studio/web/assets/{Chat-4RrB5134.js → Chat-D3j-1yY1.js} +1 -1
  604. package/studio/web/assets/{FloatingChat-omlNMDcJ.js → FloatingChat-BxJGYcfB.js} +3 -3
  605. package/studio/web/assets/{apiToken-Bke3wvfZ.js → apiToken-CT55oWOe.js} +1 -1
  606. package/studio/web/assets/{applications-DvFDYJgK.js → applications-D9Ct9_vm.js} +1 -1
  607. package/studio/web/assets/{cssMode-C1vRa7zh.js → cssMode-DV8H7VwA.js} +1 -1
  608. package/studio/web/assets/{editor-3XRWEDWX.js → editor-uatc0unt.js} +1 -1
  609. package/studio/web/assets/{html-CTY5tdMr.js → html-Bm6D6paN.js} +1 -1
  610. package/studio/web/assets/{htmlMode-CG1vSD9t.js → htmlMode-CEn7tpLG.js} +1 -1
  611. package/studio/web/assets/{index-Dfpeofdu.js → index-BIXW6Pu4.js} +5 -5
  612. package/studio/web/assets/{index.lazy-B1VOIv-t.js → index.lazy-UI7L-Vrk.js} +1 -1
  613. package/studio/web/assets/{javascript-CJeJzGnI.js → javascript-CJ0G3AFZ.js} +1 -1
  614. package/studio/web/assets/{jsonMode-CYPBwM82.js → jsonMode-DQADAYEa.js} +1 -1
  615. package/studio/web/assets/{languageServices-IKH4GUHl.js → languageServices-CAQJXWcI.js} +1 -1
  616. package/studio/web/assets/{lspLanguageFeatures-BC8_gxKG.js → lspLanguageFeatures-CCQ8P5sY.js} +1 -1
  617. package/studio/web/assets/{notifications-BfKBpYcq.js → notifications-BbxTU6Aw.js} +1 -1
  618. package/studio/web/assets/{notifications-DFbArTfC.js → notifications-Cvb3P1lB.js} +1 -1
  619. package/studio/web/assets/{profile-Q4-T6c-S.js → profile-Yyb7gsvL.js} +1 -1
  620. package/studio/web/assets/{regions-CtkV0xje.js → regions-OgjGHlU5.js} +1 -1
  621. package/studio/web/assets/{register-1ZZuMsiA.js → register-6qwNEOY3.js} +2 -2
  622. package/studio/web/assets/{setComponentFile-ZWLcWv5X.js → setComponentFile-BilDMtgB.js} +1 -1
  623. package/studio/web/assets/{setup-DWprJyJy.js → setup-J6qJ7OIU.js} +2 -2
  624. package/studio/web/assets/{status--aNm8isn.js → status-0RWGcfyD.js} +1 -1
  625. package/studio/web/assets/{toggleHighContrast-Cq_lt3XD.js → toggleHighContrast-BIn-vErT.js} +1 -1
  626. package/studio/web/assets/{tsMode-pjgytARx.js → tsMode-DgUXku4d.js} +1 -1
  627. package/studio/web/assets/{typescript-Cdg0mqUh.js → typescript-C9orXcsM.js} +1 -1
  628. package/studio/web/assets/{useEntityRestURL-RZhaY8Rn.js → useEntityRestURL-BEoXXbUB.js} +1 -1
  629. package/studio/web/assets/{workers-BgoXIqQe.js → workers-JVzSDmgx.js} +1 -1
  630. package/studio/web/assets/{xml-BwSeDMiP.js → xml-Cq-S8S4X.js} +1 -1
  631. package/studio/web/assets/{yaml-DotCUG5l.js → yaml-sfoRdh1M.js} +1 -1
  632. package/studio/web/index.html +1 -1
  633. package/upgrade/directives/5-3-0.ts +132 -0
  634. package/upgrade/directives/directivesController.ts +2 -1
  635. package/utility/OperationFunctionCaller.ts +2 -1
  636. package/utility/common_utils.ts +30 -5
  637. package/utility/componentNames.ts +12 -0
  638. package/utility/environment/environmentManager.ts +3 -7
  639. package/utility/environment/systemInformation.ts +7 -0
  640. package/utility/errors/commonErrors.ts +4 -0
  641. package/utility/errors/hdbError.ts +57 -0
  642. package/utility/hdbTerms.ts +19 -0
  643. package/utility/logging/harper_logger.ts +278 -26
  644. package/utility/logging/logGenerationCoordinator.ts +196 -0
  645. package/utility/logging/logRotation.ts +367 -0
  646. package/utility/logging/logRotator.ts +213 -81
  647. package/utility/nodeIdentity.ts +45 -0
  648. package/utility/npmUtilities.ts +12 -8
  649. package/utility/operationPermissions.ts +18 -1
  650. package/utility/operation_authorization.ts +231 -42
  651. package/utility/watchPath.ts +63 -0
  652. package/utility/watcherFallback.ts +148 -0
  653. package/validation/configValidator.ts +215 -75
  654. package/validation/installValidator.ts +15 -0
  655. package/validation/validationWrapper.ts +18 -4
@@ -0,0 +1,3043 @@
1
+ import { performance } from 'node:perf_hooks';
2
+ import { Packr } from 'msgpackr';
3
+ import harperLogger from '../utility/logging/harper_logger.ts';
4
+ import { ClientError, LockUnavailableError } from '../utility/errors/hdbError.ts';
5
+ import { MAX_LOCK_LEASE_MS, MIN_LOCK_LEASE_MS } from './recordLock.ts';
6
+
7
+ /**
8
+ * Cluster-wide record locks (harper#483, Phase 1): amortized per-record ownership.
9
+ * Design note: `docs/record-lock-ownership.md`.
10
+ *
11
+ * `Table.lock()` acquires the node's rocksdb-js key lock first, which bounds this process to one
12
+ * outstanding acquisition per key; only then does it run the cluster step here. That step has three
13
+ * levels at three very different rates:
14
+ *
15
+ * - **The home map** — `(generation, homes[])`, published by an operator through harper-pro and
16
+ * handed to core through `transport.homeMap()`. It is immutable for the life of its generation:
17
+ * core never computes it, never advances it, and never proceeds without it.
18
+ * - **The home node** — within a generation, a key's arbiter is a rendezvous hash over `homes[]`
19
+ * (§4.4). One arbiter per key is trivially exclusive, so there is no grant state machine at all:
20
+ * no deferral queues, no `(tsR, nodeId)` tiebreak, no synthesized grants, no split votes, no
21
+ * revocation protocol between peers.
22
+ * - **The delegation** — the exclusive right to admit critical sections on one key for a bounded
23
+ * time. While one is live, `lock()`/`unlock()` are pure Phase 0: the local key lock and **zero
24
+ * cluster messages**. Releasing the application lock does not release the delegation, so a node
25
+ * writing the same record repeatedly pays one round and then nothing.
26
+ *
27
+ * Two properties carry the safety argument, and both are enforced rather than assumed:
28
+ *
29
+ * - **One delegate per key per home.** A home never has two live delegations for a key, and a
30
+ * successor delegation is issued only after the predecessor's has been recalled-and-drained or has
31
+ * provably expired on the home's own clock plus skew. Every expiry decision on both sides is made
32
+ * on that side's monotonic clock; no remote timestamp is ever compared against a local one.
33
+ * - **A delegation bounds every handle it admitted.** An admission may not outlive its delegation, so
34
+ * recall revokes capability rather than merely closing the door (§6): the commit-time lease fence
35
+ * in `DatabaseTransaction` rejects a staged write whose handle has expired, and a recall expires
36
+ * those handles before the release is acknowledged.
37
+ *
38
+ * `nodeId` here is the globally stable node NAME. It is deliberately not the audit entry's `nodeId`:
39
+ * `nodeIdMapping.ts` hands out per-node short ids (0 is always local), so the same node has different
40
+ * ids on different nodes and any ordering built on them would order the same pair differently on two
41
+ * nodes.
42
+ *
43
+ * Successor freshness follows §7: a clean release carries inherited origin-log dependencies, the
44
+ * home advances the releasing origin to that entry's own position, and the next delegate cannot
45
+ * admit until its transport has made the set visible. Missing lineage takes the weaker recovery
46
+ * barrier and fails closed if that barrier cannot be established.
47
+ */
48
+
49
+ /**
50
+ * How long a home waits before re-sending a recall that FAILED. A recall the delegate confirmed is
51
+ * never re-sent: it has stopped admitting, and the grant is then cleared by its release or by its own
52
+ * deadline. Without both rules a contender polling at 25 ms re-armed the recall on every pass and
53
+ * turned one handoff into an RPC storm lasting the rest of the delegation.
54
+ */
55
+ export const RECALL_RETRY_MS = 1_000;
56
+
57
+ /** Margin a home adds to a delegation it issued, so the delegate always stops admitting first. */
58
+ export const LOCK_LEASE_SKEW_MS = 5_000;
59
+ /**
60
+ * How long a delegation runs, independent of any one caller's lock lease. It MUST be longer than the
61
+ * longest lease it will admit, or the amortization does not exist: a delegation sized to the caller's
62
+ * lease has no room left for the next lock, so every repeat `lock()` renews and pays a round trip —
63
+ * exactly the cost this design is built to remove.
64
+ */
65
+ export const DELEGATION_LEASE_MS = MAX_LOCK_LEASE_MS + 60_000;
66
+ /**
67
+ * Below this an admission map is too small for its dead entries to matter, so the expiry sweep in
68
+ * `#pruneAdmissions` does not run at all; above it the map may reach twice the live set first.
69
+ */
70
+ const ADMISSION_SWEEP_FLOOR = 64;
71
+ const TICK_INTERVAL_MS = 100;
72
+ /**
73
+ * How often `tick()` asks the transport whether this thread still coordinates. Far coarser than the
74
+ * tick because the answer only has to be sampled faster than a gap can matter, and it cannot matter
75
+ * below a full delegation lease: whatever coordinated during the gap is inside its own quarantine
76
+ * until then. At the tick rate an idle coordinating table would call the transport 10 times a second
77
+ * for the life of the process.
78
+ */
79
+ const OWNERSHIP_POLL_MS = 1_000;
80
+ const WARN_INTERVAL_MS = 60_000;
81
+ /**
82
+ * Bounds the grants ONE TABLE's coordinator can accumulate — a home may never forget a delegation
83
+ * before its expiry, so the only bound available is a refusal to issue more. There is one coordinator
84
+ * per table, so the process-wide exposure is this times the number of tables under lock pressure; a
85
+ * true process-level bound is harper#2581 and is sized with the enablement measurements.
86
+ */
87
+ const MAX_DELEGATIONS_PER_TABLE = 10_000;
88
+ /** Bounds what a single peer can make one table's home retain, so one node cannot exhaust it. */
89
+ const MAX_DELEGATIONS_PER_REQUESTER = 2_000;
90
+ /** Bounds expiry work per tick, so a burst of expiries cannot stall the event loop. */
91
+ const MAX_EXPIRIES_PER_TICK = 256;
92
+ /** How many entries a relay sweep may LOOK at per tick, so a thread holding many live relayed
93
+ * admissions does not walk all of them every 100 ms to find nothing. Well above any realistic count of
94
+ * concurrent off-owner locks on one thread, so an ordinary sweep still completes a full pass. */
95
+ const MAX_EXAMINED_PER_TICK = 4_096;
96
+ /** How long past its lease a relayed admission entry is kept before pruning (harper-pro#852), so the
97
+ * handle's own lease timer fires first and forwards its release to the owner rather than racing the
98
+ * sweep that would drop the entry it needs. A few ticks is plenty. */
99
+ const REMOTE_PRUNE_GRACE_MS = 500;
100
+ /**
101
+ * What an owner-worker acquire (harper-pro#852) reserves out of the caller's `waitMs` for the round
102
+ * trip, so the transport is asked for a wait it can answer WITHIN the caller's budget. `lock()` holds
103
+ * the native key for the whole wait, so the hop must come out of that budget, never on top of it:
104
+ * overshooting blocks every other worker on the key for the overshoot. Halved for a caller whose wait
105
+ * is shorter than the allowance, so a short wait still leaves the owner something to work with.
106
+ *
107
+ * Small on purpose. This covers only the core-to-transport boundary: a transport that relays bounds
108
+ * itself inside the wait it is handed and reserves its own margin for the hops it makes, so reserving
109
+ * a hop-sized allowance here as well would subtract the same round trip from the caller twice.
110
+ */
111
+ const REMOTE_ACQUIRE_HOP_MS = 500;
112
+ /** Core's last-resort net on an owner-worker acquire, on top of the caller's `waitMs`. The transport
113
+ * already bounds itself inside that budget, so this fires only when it never returns at all — small,
114
+ * because it is a wedged-transport net and not a second full wait. */
115
+ const REMOTE_ACQUIRE_BACKSTOP_MS = 1_000;
116
+ const MAX_NODE_NAME_LENGTH = 255;
117
+ const MAX_LOCK_DEPENDENCIES = 1_024;
118
+ const MAX_DEPENDENCY_SETS_PER_TABLE = 20_000;
119
+ /**
120
+ * A 256 KiB filter keeps the false-positive rate near 1% through 200,000 distinct delegated keys
121
+ * per lock-active table and generation. Saturation remains safe — it selects recovery — but making
122
+ * that the normal path would turn every first lock on a new key into a cluster-wide barrier.
123
+ */
124
+ const DELEGATED_KEY_FILTER_WORDS = 65_536;
125
+ /** A node whose identity resolved to one of these is not distinctive enough to be a ring member. */
126
+ const NON_DISTINCTIVE_NODE_NAMES = new Set(['127.0.0.1', '::1', 'localhost', '0.0.0.0']);
127
+
128
+ /**
129
+ * The two control entries. `lockRequest`/`lockGrant` belonged to the Ricart–Agrawala rule the
130
+ * design note replaces; they never shipped enabled, so their nibbles were retired rather than
131
+ * migrated (`auditStore.ts`). Delegation request/grant/recall are unicast over the transport, not
132
+ * entries — the release stays on the replicated log because it is what orders a handoff behind the
133
+ * delegate's own data writes, and the barrier is on it because being replicated is its whole
134
+ * purpose (§7.2): a member commits one on request, and its position is the point a peer must have
135
+ * applied that origin through before a recovery-mode successor may admit.
136
+ */
137
+ export type LockControlType = 'lockRelease' | 'lockBarrier';
138
+
139
+ /**
140
+ * The operator-agreed map a key's home is derived from. Supplied by harper-pro; core never computes
141
+ * it and never advances it. Immutable for the life of a generation — nothing a node observes changes
142
+ * it, which is why no agreement protocol runs here (§4).
143
+ */
144
+ export interface LockHomeMap {
145
+ /** Monotonic per database. Part of the fencing token, so it must never go backwards. */
146
+ generation: number;
147
+ /**
148
+ * Every node that participates in cluster record locks for this database — not only the ones an
149
+ * operator thinks of as arbiters. Order is irrelevant; the ring hashes each name independently.
150
+ *
151
+ * It is one set and not two because a home refuses a delegation to any node this list does not
152
+ * name (that is what keeps a decommissioned node from taking one), so a node absent from it can
153
+ * neither home a key nor lock one. Rendezvous hashing then makes every listed node the arbiter for
154
+ * its share of the ring, which is the property that costs a second list nothing.
155
+ */
156
+ homes: string[];
157
+ /**
158
+ * This node's durably persisted, monotonic incarnation counter as a home (§5.1). A random value
159
+ * makes a stale reply identifiable but not ORDERABLE: a home that restarts and re-issues counter 1
160
+ * after having issued counter 50 would let a delayed counter-50 write defeat its successor.
161
+ *
162
+ * **Per coordination incarnation, not per process.** Coordinator state — including the delegation
163
+ * counter — is per-thread, so a replacement coordinating worker starts counting from zero. If the
164
+ * incarnation did not advance with it, the new worker would re-mint tokens its predecessor already
165
+ * issued, and §4.3's incarnation-bound quiescence acknowledgements would survive a restart that
166
+ * discarded everything they attested to.
167
+ */
168
+ homeIncarnation: number;
169
+ }
170
+
171
+ /**
172
+ * A delegation's fencing token, ordered lexicographically as
173
+ * `(generation, homeIncarnation, counter)`. Comparable across homes only within a generation, which
174
+ * is all that is needed: a key has exactly one home per generation.
175
+ */
176
+ export type FencingToken = readonly [generation: number, homeIncarnation: number, counter: number];
177
+ export type LockDependency = readonly [origin: string, position: number];
178
+ export type LockDependencySet = readonly LockDependency[];
179
+
180
+ export function compareTokens(a: FencingToken, b: FencingToken): number {
181
+ return a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
182
+ }
183
+
184
+ function isFencingToken(value: unknown): value is FencingToken {
185
+ return (
186
+ Array.isArray(value) &&
187
+ value.length === 3 &&
188
+ value.every((part) => typeof part === 'number' && Number.isFinite(part))
189
+ );
190
+ }
191
+
192
+ export interface LockReleaseEntry {
193
+ type: 'lockRelease';
194
+ /** The locked record's id. Control entries carry it here, never as the audit entry's recordId. */
195
+ key: any;
196
+ /** Node that held the delegation being released. */
197
+ requester: string;
198
+ /**
199
+ * The released delegation's whole fencing token. The counter alone is NOT enough to identify it: a
200
+ * home that restarts, or whose coordinator is recreated, begins counting again, so a delayed
201
+ * release from a previous incarnation would match a live grant's counter and clear it while its
202
+ * delegate is still admitting.
203
+ */
204
+ token: FencingToken;
205
+ /**
206
+ * Inherited clean-handoff lineage. Null hands back an unused first grant without advancing it; a
207
+ * renewed grant still drops retained lineage because its earlier token may have admitted writes.
208
+ * Absent means legacy/unknown.
209
+ */
210
+ dependencies?: LockDependencySet | null;
211
+ }
212
+
213
+ /**
214
+ * The §7.2 recovery fence. It names no key and no token: it is appended after every transaction the
215
+ * writing node had committed when it was requested, so a peer that has applied that origin's log
216
+ * through this entry has applied all of them. The coordinator never acts on one.
217
+ */
218
+ export interface LockBarrierEntry {
219
+ type: 'lockBarrier';
220
+ /** Supplied by the requesting transport; distinguishes barriers an origin stamped identically across a restart. */
221
+ nonce: number;
222
+ }
223
+
224
+ export type LockControlEntry = LockReleaseEntry | LockBarrierEntry;
225
+
226
+ /**
227
+ * What a home replies to a delegation request. One shape rather than a discriminated union: the
228
+ * repo compiles with `strict: false`, where TypeScript does not narrow a union on a boolean literal,
229
+ * so a union here would type-check the denial fields as absent on every branch and then not enforce
230
+ * it. `granted` says which half is populated.
231
+ */
232
+ export interface DelegationReply {
233
+ granted: boolean;
234
+ /** Granted only. */
235
+ token?: FencingToken;
236
+ /** Granted only. How long the delegate may admit for, as a DURATION — never a remote clock reading. */
237
+ leaseMs?: number;
238
+ /** Granted only. Null selects the recovery barrier; an array is the exact clean-handoff fence. */
239
+ dependencies?: LockDependencySet | null;
240
+ /**
241
+ * Denied only. `contended` is the one reason that means another node holds the key, and so the only
242
+ * one a timeout may report as 423. `generation` (the two sides hold different home maps),
243
+ * `unknown-node` (this node is not named in the map), `quarantine` (the home is inside its §4.3
244
+ * restart interval) and `timeout` (the home never answered) all describe something other than
245
+ * contention, so each ends as a retryable 503 rather than telling the caller a key nobody holds is
246
+ * held.
247
+ */
248
+ reason?: 'contended' | 'generation' | 'unknown-node' | 'capacity' | 'not-home' | 'quarantine' | 'timeout';
249
+ /** Denied with `generation`, so a stale requester can re-derive the ring without another round trip. */
250
+ generation?: number;
251
+ retryAfterMs?: number;
252
+ }
253
+
254
+ export interface DelegationRequest {
255
+ key: any;
256
+ /** The asking node. Established by the transport, never read from an untrusted payload. */
257
+ requester: string;
258
+ generation: number;
259
+ leaseMs: number;
260
+ }
261
+
262
+ export interface DelegationRecall {
263
+ key: any;
264
+ token: FencingToken;
265
+ }
266
+
267
+ /**
268
+ * Supplied by harper-pro. Core never computes cluster topology; it only refuses to promise a
269
+ * cluster-wide lock that this contract cannot back.
270
+ */
271
+ export interface ClusterLockTransport {
272
+ /**
273
+ * The operator-agreed home map for the database, or undefined while none is available — before the
274
+ * node has the current generation, or while peers disagree about its digest. Core fails closed on
275
+ * undefined rather than guessing a ring.
276
+ *
277
+ * **One obligation core cannot check, and relies on** (§4.3): a generation change is
278
+ * operator-sequenced, so `g+1` may only be answered here once the control plane's one-shot
279
+ * activation record is active — every old home quiesced (each acknowledgement bound to the
280
+ * acknowledger's `homeIncarnation`, so a restart during the drain invalidates it) or externally
281
+ * fenced, then `DELEGATION_LEASE_MS + skew` elapsed. Core cannot observe what happened on other
282
+ * nodes; the operator can.
283
+ *
284
+ * The other interval — a restart of *this* process — is core's own, because a generation does not
285
+ * advance on a restart and there is no external event to hang it on. See `#grantableAfterMono`.
286
+ */
287
+ homeMap(database: string): LockHomeMap | undefined;
288
+ /**
289
+ * Overrides core's cold-start grant quarantine (§4.3). Set it only where a previous incarnation
290
+ * of this process provably issued nothing — a fresh database, a first start, or a test. Because the
291
+ * same attestation enables the virgin-key freshness fast path, it must prove that no earlier
292
+ * incarnation delegated any key under the current home-map generation at any time, not merely that
293
+ * its last delegation lease has elapsed. Omitted means core enforces the full
294
+ * `DELEGATION_LEASE_MS + skew` from that coordinator's construction — see `#grantableAfterMono` for
295
+ * why neither process start nor thread start is a sound anchor.
296
+ */
297
+ grantableAfterMono?: number;
298
+ /**
299
+ * Whether this worker thread owns lock coordination for the process. Coordinator state is
300
+ * per-thread while the key lock it arbitrates is process-wide, so a second thread running its own
301
+ * delegations would arbitrate against a different view. Core fails closed off the owner thread.
302
+ */
303
+ ownsCoordination(): boolean;
304
+ /** Ask `node` (the key's home) for a delegation. Unicast; rejects if the node is unreachable. */
305
+ requestDelegation(
306
+ node: string,
307
+ database: string,
308
+ table: string,
309
+ request: DelegationRequest
310
+ ): Promise<DelegationReply>;
311
+ /** Home → delegate. Resolves once the delegate has drained and stopped admitting. */
312
+ recallDelegation(node: string, database: string, table: string, recall: DelegationRecall): Promise<void>;
313
+ /**
314
+ * Obtain an admission from the worker thread that coordinates this database, for a `lock()` served
315
+ * on a thread that is not the owner. Present only when the transport can relay across threads
316
+ * (harper-pro#852); a transport without it makes `acquire()` fail closed off the owner thread, as
317
+ * before. The returned `LockRound` was minted by the owner's coordinator; the caller's coordinator
318
+ * records it as a REMOTE admission and installs the handle's revoker locally, so a recall on the
319
+ * owner fences a write this thread's handle staged. `mintedMono` is comparable across threads
320
+ * because `performance.now()` shares one time origin process-wide.
321
+ */
322
+ acquireOnOwner?(database: string, table: string, key: any, leaseMs: number, waitMs: number): Promise<LockRound>;
323
+ /** Release a remote admission on the owner thread (the counterpart of `acquireOnOwner`). */
324
+ releaseOnOwner?(database: string, table: string, key: any, admissionId: number): Promise<void> | void;
325
+ /**
326
+ * Establish an exact clean-handoff dependency set, or recover the strongest reachable-member
327
+ * barrier when `dependencies` is null. Recovery returns the captured positions for current lock
328
+ * participants that were made visible; it may drain additional replication peers without carrying
329
+ * them in the key's lineage. Clean waits may return void. Concurrent recovery snapshots should be
330
+ * coalesced.
331
+ *
332
+ * Recovery asks each reachable member for a `lockBarrier` entry (`writeLockBarrier`) and drains
333
+ * that member's stream through the position it returns. Core races this promise against the
334
+ * lock's own deadline but cannot cancel it; `deadlineMs` is the wait remaining at the call, so the
335
+ * transport can bound its own work to it instead of outliving the lock that asked.
336
+ */
337
+ establishLockFreshness(
338
+ database: string,
339
+ table: string,
340
+ key: any,
341
+ dependencies: LockDependencySet | null,
342
+ deadlineMs: number
343
+ ): Promise<LockDependencySet | void>;
344
+ /** Emit a control entry and return the committed entry's local origin-log position when available. */
345
+ writeControl?(table: string, entry: LockControlEntry): Promise<number | void> | number | void;
346
+ /**
347
+ * Assigned at registration so a transport can push a received entry in directly. `author` and
348
+ * `position` come from the authenticated origin-log header, never from the payload.
349
+ */
350
+ onControlEntry?(database: string, table: string, entry: LockControlEntry, author: string, position: number): void;
351
+ /** Assigned at registration. Inbound delegation request from a peer, for a key this node homes. */
352
+ onDelegationRequest?(database: string, table: string, request: DelegationRequest): Promise<DelegationReply>;
353
+ /** Assigned at registration. Inbound recall from a key's home. */
354
+ onDelegationRecall?(database: string, table: string, recall: DelegationRecall): Promise<void>;
355
+ }
356
+
357
+ // A private structure dictionary, so a control payload can never contribute to — or depend on — the
358
+ // table's own, and never passes through schema projection (see writeLockControlEntry in Table.ts).
359
+ // Record mode stays ON deliberately: the reader is the receiving table's decoder (via
360
+ // `auditRecord.getValue`), which repurposes a range of positive fixints as structure ids. A payload
361
+ // packed without record mode writes an integer record key of 64..127 as a bare fixint, which that
362
+ // decoder then reads as a structure header and the whole entry fails to decode.
363
+ let controlStructures: unknown[] = [];
364
+ let controlPackr = new Packr({ structures: controlStructures });
365
+
366
+ export function encodeLockControlPayload(entry: LockControlEntry): Uint8Array {
367
+ if (entry.type === 'lockBarrier') return controlPackr.pack([1, entry.nonce]);
368
+ const [generation, homeIncarnation, counter] = entry.token;
369
+ if (entry.dependencies === undefined)
370
+ return controlPackr.pack([entry.key, entry.requester, generation, homeIncarnation, counter]);
371
+ return controlPackr.pack([1, entry.key, entry.requester, generation, homeIncarnation, counter, entry.dependencies]);
372
+ }
373
+
374
+ function isNodeName(value: unknown): value is string {
375
+ return typeof value === 'string' && value.length > 0 && value.length <= MAX_NODE_NAME_LENGTH;
376
+ }
377
+
378
+ function isDuration(value: unknown, min: number, max: number): value is number {
379
+ return typeof value === 'number' && Number.isFinite(value) && value >= min && value <= max;
380
+ }
381
+
382
+ /**
383
+ * Accept exactly what `ordered-binary` encodes, because that is what `keyIdOf` runs on the key.
384
+ * Refusing a shape the encoder handles makes the home answer `not-home` for a key nobody holds, and
385
+ * the requester retries that to its own 423. A `Uint8Array` returns from the unpack as a `Buffer`
386
+ * and a small `bigint` as a `number`; both encode to the same stored key, which is the identity at
387
+ * issue rather than the JS value.
388
+ */
389
+ function isEncodableKey(value: unknown): boolean {
390
+ const type = typeof value;
391
+ if (type === 'string' || type === 'bigint' || type === 'number' || type === 'boolean') return true;
392
+ if (value === null || value instanceof Uint8Array) return true;
393
+ return Array.isArray(value) && (value as unknown[]).every(isEncodableKey);
394
+ }
395
+
396
+ /**
397
+ * Decode a received control payload, or undefined when it is not one this version understands. The
398
+ * tuple length is validated exactly: a future version that grows the payload must bump the type
399
+ * rather than widen this one, since a partially-understood release would clear a delegation on terms
400
+ * the sender did not intend.
401
+ */
402
+ export function decodeLockControlPayload(type: unknown, value: unknown): LockControlEntry | undefined {
403
+ if (type !== 'lockRelease' && type !== 'lockBarrier') return undefined;
404
+ let tuple: unknown;
405
+ try {
406
+ tuple = value instanceof Uint8Array ? controlPackr.unpack(value) : value;
407
+ } catch {
408
+ return undefined;
409
+ }
410
+ if (!Array.isArray(tuple)) return undefined;
411
+ if (type === 'lockBarrier') {
412
+ if (tuple.length !== 2 || tuple[0] !== 1) return undefined;
413
+ const nonce = tuple[1];
414
+ return typeof nonce === 'number' && Number.isFinite(nonce) ? { type: 'lockBarrier', nonce } : undefined;
415
+ }
416
+ if (tuple.length !== 5 && tuple.length !== 7) return undefined;
417
+ try {
418
+ return decodeTuple(tuple);
419
+ } catch {
420
+ // isEncodableKey recurses; a deeply nested array in a peer or replayed payload would otherwise
421
+ // raise a RangeError into the replicated apply loop instead of being dropped as malformed.
422
+ return undefined;
423
+ }
424
+ }
425
+
426
+ function decodeTuple(tuple: unknown[]): LockReleaseEntry | undefined {
427
+ const versioned = tuple.length === 7;
428
+ if (versioned && tuple[0] !== 1) return undefined;
429
+ const offset = versioned ? 1 : 0;
430
+ const [key, requester, generation, homeIncarnation, counter] = tuple.slice(offset, offset + 5);
431
+ if (!isEncodableKey(key) || !isNodeName(requester)) return undefined;
432
+ for (const part of [generation, homeIncarnation, counter])
433
+ if (typeof part !== 'number' || !Number.isFinite(part)) return undefined;
434
+ let dependencies: LockDependencySet | null | undefined;
435
+ if (versioned) {
436
+ const rawDependencies = tuple[6];
437
+ if (rawDependencies === null) dependencies = null;
438
+ else dependencies = normalizeDependencies(rawDependencies);
439
+ }
440
+ const entry: LockReleaseEntry = {
441
+ type: 'lockRelease',
442
+ key,
443
+ requester,
444
+ token: [generation, homeIncarnation, counter] as FencingToken,
445
+ };
446
+ if (versioned) entry.dependencies = dependencies;
447
+ return entry;
448
+ }
449
+
450
+ function normalizeDependencies(value: unknown, homes?: readonly string[]): LockDependencySet | undefined {
451
+ if (!Array.isArray(value) || value.length > MAX_LOCK_DEPENDENCIES) return undefined;
452
+ const positions = new Map<string, number>();
453
+ const allowedOrigins = homes && new Set(homes);
454
+ for (const dependency of value) {
455
+ if (!Array.isArray(dependency) || dependency.length !== 2) return undefined;
456
+ const [origin, position] = dependency;
457
+ if (!isNodeName(origin) || typeof position !== 'number' || !Number.isFinite(position) || position < 0)
458
+ return undefined;
459
+ if (allowedOrigins && !allowedOrigins.has(origin)) return undefined;
460
+ if (positions.has(origin)) return undefined;
461
+ positions.set(origin, position);
462
+ }
463
+ return [...positions].sort(([a], [b]) => a.localeCompare(b));
464
+ }
465
+
466
+ class DelegatedKeyFilter {
467
+ #bits = new Uint32Array(DELEGATED_KEY_FILTER_WORDS);
468
+
469
+ add(key: unknown): void {
470
+ const [first, second] = this.#hashes(key);
471
+ for (let index = 0; index < 4; index++) this.#set((first + Math.imul(index, second)) >>> 0);
472
+ }
473
+
474
+ has(key: unknown): boolean {
475
+ const [first, second] = this.#hashes(key);
476
+ for (let index = 0; index < 4; index++) if (!this.#get((first + Math.imul(index, second)) >>> 0)) return false;
477
+ return true;
478
+ }
479
+
480
+ clear(): void {
481
+ this.#bits.fill(0);
482
+ }
483
+
484
+ copyFrom(other: DelegatedKeyFilter): void {
485
+ this.#bits.set(other.#bits);
486
+ }
487
+
488
+ #set(hash: number): void {
489
+ const bit = hash % (this.#bits.length * 32);
490
+ this.#bits[bit >>> 5] |= 1 << (bit & 31);
491
+ }
492
+
493
+ #get(hash: number): boolean {
494
+ const bit = hash % (this.#bits.length * 32);
495
+ return (this.#bits[bit >>> 5] & (1 << (bit & 31))) !== 0;
496
+ }
497
+
498
+ #hashes(key: unknown): [number, number] {
499
+ const value = `${typeof key}:${String(key)}`;
500
+ let first = 0x811c9dc5;
501
+ let second = 0x9e3779b9;
502
+ for (let index = 0; index < value.length; index++) {
503
+ first = Math.imul(first ^ value.charCodeAt(index), 0x01000193);
504
+ second = Math.imul(second ^ value.charCodeAt(index), 0x85ebca6b);
505
+ }
506
+ return [first >>> 0, (second | 1) >>> 0];
507
+ }
508
+ }
509
+
510
+ /**
511
+ * Rendezvous (highest-random-weight) hash: a key's home is the node with the greatest score for that
512
+ * key. Chosen over a modulo of a key hash because a generation change moves only the keys homed on a
513
+ * departing node, rather than re-homing the whole space — which matters because every re-homed key
514
+ * pays the §7.2 recovery path on its next lock.
515
+ *
516
+ * The hash is FNV-1a over the member name and the key's stable id. It does not need to be
517
+ * cryptographic: it is not a defense against anything, only a deterministic agreement between nodes
518
+ * that already agree on `homes`.
519
+ */
520
+ function scoreFor(member: string, keyId: string): number {
521
+ let hash = 0x811c9dc5;
522
+ for (let i = 0; i < member.length; i++) {
523
+ hash ^= member.charCodeAt(i);
524
+ hash = Math.imul(hash, 0x01000193);
525
+ }
526
+ // A separator that cannot appear in either operand, so ("ab","c") and ("a","bc") cannot collide.
527
+ hash ^= 0xff;
528
+ hash = Math.imul(hash, 0x01000193);
529
+ for (let i = 0; i < keyId.length; i++) {
530
+ hash ^= keyId.charCodeAt(i);
531
+ hash = Math.imul(hash, 0x01000193);
532
+ }
533
+ return hash >>> 0;
534
+ }
535
+
536
+ /**
537
+ * The string the ring hashes for a key. §4.4 scopes it by database and table: hashing the record id
538
+ * alone would home id `42` in every table of every database on the same node, concentrating unrelated
539
+ * hot keys on one arbiter. The separator cannot appear in a name or a stringified key id.
540
+ */
541
+ export function ringKeyFor(database: string, table: string, keyId: unknown): string {
542
+ return `${database}\u0000${table}\u0000${String(keyId)}`;
543
+ }
544
+
545
+ export function homeFor(keyId: string, homes: string[]): string | undefined {
546
+ let best: string | undefined;
547
+ let bestScore = -1;
548
+ for (const home of homes) {
549
+ const score = scoreFor(home, keyId);
550
+ // Ties break on the name so every node picks the same home from the same set.
551
+ if (score > bestScore || (score === bestScore && best !== undefined && home > best)) {
552
+ bestScore = score;
553
+ best = home;
554
+ }
555
+ }
556
+ return best;
557
+ }
558
+
559
+ /** A delegation this node holds: the right to admit critical sections on one key. */
560
+ interface Delegation {
561
+ key: any;
562
+ token: FencingToken;
563
+ dependencies: LockDependencySet;
564
+ /** Monotonic deadline on THIS node. A delegate stops admitting here. */
565
+ expiresMono: number;
566
+ /** Set by a recall. No new admission may start, but live ones are drained first. */
567
+ recalled: boolean;
568
+ /**
569
+ * Every admission this delegation is still answerable for, by id. An entry stays here after its
570
+ * caller unlocks: §6 revokes CAPABILITY, not admission. A caller that staged a write and then
571
+ * called `unlock()` has nothing left to wait on, but its write is still uncommitted and would land
572
+ * after the successor was admitted — so surrender has to fence it, which means keeping its revoker
573
+ * until its own lease runs out or authority is lost.
574
+ *
575
+ * Addressed by id through `#admissions` rather than owned by the delegation OBJECT, so a handle
576
+ * survives a renewal with its delegation and is revoked with it when authority is actually lost.
577
+ */
578
+ admissions: Map<number, Admission>;
579
+ /**
580
+ * How many of those admissions have not unlocked yet. A drain waits for THIS to reach zero — an
581
+ * unlocked-but-staged write is revoked rather than waited for (harper#2580).
582
+ */
583
+ holding: number;
584
+ /** `admissions.size` at which the next full expiry sweep runs; see `#pruneAdmissions`. */
585
+ sweepAtSize: number;
586
+ /** Resolvers waiting for the drain to finish, so recall can reply once rather than poll. */
587
+ drained?: (() => void)[];
588
+ /** The one in-flight `#surrender` for this delegation; see there for why it is memoized. */
589
+ surrendering?: Promise<void>;
590
+ }
591
+
592
+ /**
593
+ * One `lock()` this thread admitted from the OWNER thread (harper-pro#852), addressed by a LOCAL id
594
+ * distinct from the owner's own admission id (see `#remoteByOwnerId`), so a stale local admission can
595
+ * never share a number with an incoming owner-minted one. `revoke` fences this thread's handle;
596
+ * `revoked` latches a revoke that landed before `registerAdmission` supplied the real revoker.
597
+ * `fenceWaiters` resolve once the handle is provably fenced — the owner waits on them before it writes
598
+ * the release, so an ack never claims a fence the handle has not actually taken.
599
+ */
600
+ interface RemoteAdmission {
601
+ /** The owner's admission id, echoed on release and matched by an inbound revoke. */
602
+ ownerAdmissionId: number;
603
+ key: any;
604
+ /** The caller-side handle fence — today always `Table.lock`'s `() => handle.revokeLease()`, which is
605
+ * synchronous and total, never the owner-side async relay revoker (that lives on a delegation
606
+ * `Admission` instead). `registerAdmission` accepts an async revoker all the same, so the ack paths
607
+ * here settle on the outcome rather than assuming this one. */
608
+ revoke: () => void;
609
+ /** Monotonic deadline of the handle's own lease; the entry is dropped once past it. */
610
+ expiresMono: number;
611
+ revoked: boolean;
612
+ fenceWaiters: (() => void)[];
613
+ }
614
+
615
+ /** One `lock()` admitted under a delegation. */
616
+ interface Admission {
617
+ /**
618
+ * Fences the handle's write capability. A no-op until `registerAdmission` supplies the real one. A
619
+ * handle admitted on another worker thread revokes over a message and resolves once that thread has
620
+ * fenced it; `#surrender` waits for that (or the admission's own lease) before writing the release.
621
+ */
622
+ revoke: () => void | Promise<void>;
623
+ /** Monotonic deadline of the handle's OWN lease, after which it fences itself and can be dropped. */
624
+ expiresMono: number;
625
+ /** False once the caller unlocked: it no longer blocks a drain, but is still revocable. */
626
+ holding: boolean;
627
+ }
628
+
629
+ /** A delegation this node issued as a key's home. */
630
+ interface HomeGrant {
631
+ key: any;
632
+ delegate: string;
633
+ token: FencingToken;
634
+ /** Requirement handed to this delegate; reused if the same node renews after losing local state. */
635
+ dependencies: LockDependencySet | null;
636
+ /** This grant advanced from an earlier token that may already have admitted writes. */
637
+ renewed?: boolean;
638
+ /**
639
+ * Monotonic deadline on THIS node, set to the delegate's lease PLUS skew. The home always outwaits
640
+ * the delegate, so it can never re-grant a key the previous delegate still believes it holds.
641
+ */
642
+ expiresMono: number;
643
+ recalling?: Promise<void>;
644
+ /** Set once the delegate confirmed it stopped admitting, so the recall is never re-sent. */
645
+ recallConfirmed?: boolean;
646
+ /** After a FAILED recall, the earliest this home may try that delegate again (`RECALL_RETRY_MS`). */
647
+ recallRetryAfterMono?: number;
648
+ }
649
+
650
+ interface PendingDelegation {
651
+ key: any;
652
+ token: FencingToken;
653
+ recalled: boolean;
654
+ recalledPromise: Promise<void>;
655
+ markRecalled: () => void;
656
+ }
657
+
658
+ const PENDING_DELEGATION_RECALLED = Symbol('pending delegation recalled');
659
+
660
+ function createPendingDelegation(key: any, token: FencingToken, recalledBeforeReply: boolean): PendingDelegation {
661
+ let resolveRecall: () => void;
662
+ const pending: PendingDelegation = {
663
+ key,
664
+ token,
665
+ recalled: false,
666
+ recalledPromise: new Promise((resolve) => (resolveRecall = resolve)),
667
+ markRecalled() {
668
+ if (pending.recalled) return;
669
+ pending.recalled = true;
670
+ resolveRecall();
671
+ },
672
+ };
673
+ if (recalledBeforeReply) pending.markRecalled();
674
+ return pending;
675
+ }
676
+
677
+ interface PendingRequest {
678
+ recalledToken?: FencingToken;
679
+ }
680
+
681
+ export interface LockRound {
682
+ /** The identity to stamp the holder's writes with. */
683
+ tsR: number;
684
+ /** The monotonic reading the admission's lease is measured from. */
685
+ mintedMono: number;
686
+ /**
687
+ * Identifies THIS admission for the life of the handle it produced. The caller hands it back on
688
+ * registration and on release. An id rather than the delegation's token, because the token changes
689
+ * on renewal while the admission does not: addressing by token made a renewed delegation lose
690
+ * track of handles it was still responsible for.
691
+ */
692
+ admissionId: number;
693
+ }
694
+
695
+ export interface LockCoordinatorOptions {
696
+ database: string;
697
+ table: string;
698
+ /** This node's globally stable name. */
699
+ nodeId: string;
700
+ transport: ClusterLockTransport;
701
+ /**
702
+ * Emit one control entry. Core passes the transport's own `writeControl` when it has one and its
703
+ * transaction-log writer otherwise, since writing to the local log IS the send.
704
+ */
705
+ writeControl: (entry: LockControlEntry) => Promise<number | void> | number | void;
706
+ /** Stable map key for a record id; core passes `writeKeyId`. */
707
+ keyIdOf: (key: any) => unknown;
708
+ /** Mints the holder's stamp; core passes the primary store's monotonic timestamp. */
709
+ nextTimestamp: () => number;
710
+ /** Monotonic clock. Every expiry decision is made on this. */
711
+ monotonic?: () => number;
712
+ skewMs?: number;
713
+ /**
714
+ * The coordinator this one replaces, when a transport is re-registered (a component reload is
715
+ * enough). Its live authority is MOVED here rather than discarded: the transport changed, but the
716
+ * handles it admitted did not, and a successor that started with an empty grant table could hand
717
+ * the same key to another node with no lease time elapsed. That is the "a home may never forget a
718
+ * grant before its expiry" rule (§8) applied across the swap rather than only within one
719
+ * coordinator's life. §11 of the design note calls for exactly this — carry live authority across,
720
+ * or fence and settle every outstanding handle before granting; carrying it across costs nothing.
721
+ */
722
+ adopt?: LockCoordinator;
723
+ /**
724
+ * Overrides the cold-start grant quarantine (§4.3). Pass `-Infinity` only where a previous
725
+ * incarnation provably issued nothing — a fresh database, or a test. Required when `monotonic` is
726
+ * an injected clock the test drives itself. See `#grantableAfterMono`.
727
+ */
728
+ grantableAfterMono?: number;
729
+ /** False in tests, which drive `tick()` themselves. */
730
+ autoTick?: boolean;
731
+ }
732
+
733
+ /**
734
+ * What a coordinator closed WITHOUT a successor leaves for its eventual replacement, per table.
735
+ *
736
+ * `close()` drops the grant table, but the delegations those grants authorize are still live on other
737
+ * nodes until their own deadlines — closing is a local event that no peer observes. A replacement
738
+ * built before then must not grant those keys again, and must not restart the counter into tokens the
739
+ * closed coordinator already issued. `#grantableAfterMono` covers the same hazard across a process
740
+ * RESTART; this covers it across an unregister and re-register inside one process, which a
741
+ * process-start reading cannot see.
742
+ */
743
+ const retiredCoordinators = new Map<string, { grantableAfterMono: number; counter: number }>();
744
+
745
+ /**
746
+ * The highest home-map generation this thread has acted under, per DATABASE — the scope the generation
747
+ * itself has, not the per-table scope a token is compared at. A generation is the high-order component
748
+ * of every fencing token (§5.1), so going backwards re-mints tokens that order BELOW ones already
749
+ * handed out, and a delayed write under the newer generation then defeats its successor. Refusing it
750
+ * database-wide is strictly stronger than refusing it per table and costs nothing: one rolled-back
751
+ * publish is one event. The map is operator-published, so the rollback route is a configuration
752
+ * restore or a partial publish rather than a protocol bug — which is why it is refused here rather
753
+ * than assumed away. Remembering it across a restart, and across threads, is harper-pro's half.
754
+ */
755
+ const highestGeneration = new Map<string, number>();
756
+
757
+ /** Bound one drain step so a single unresponsive delegate cannot consume the whole transition budget. */
758
+ function withDeadline<T>(work: Promise<T>, ms: number, message = 'the quiesce deadline elapsed'): Promise<T> {
759
+ if (!(ms > 0)) return Promise.reject(new Error(message));
760
+ const timer = delay(ms);
761
+ work.then(
762
+ () => timer.cancel(),
763
+ () => timer.cancel()
764
+ );
765
+ return Promise.race([
766
+ work,
767
+ timer.promise.then<T>(() => {
768
+ throw new Error(message);
769
+ }),
770
+ ]);
771
+ }
772
+
773
+ const tickingCoordinators = new Set<LockCoordinator>();
774
+ /**
775
+ * Every coordinator alive on this thread, so a membership transition can quiesce a whole database
776
+ * rather than one table at a time (harper-pro#856). The per-`(database, table)` resolvers cannot
777
+ * enumerate: they answer a name you already have.
778
+ */
779
+ const liveCoordinators = new Set<LockCoordinator>();
780
+ let tickTimer: ReturnType<typeof setInterval> | undefined;
781
+ function ensureTicking() {
782
+ if (tickTimer || tickingCoordinators.size === 0) return;
783
+ tickTimer = setInterval(() => {
784
+ for (const coordinator of tickingCoordinators) {
785
+ try {
786
+ coordinator.tick();
787
+ } catch (error) {
788
+ warnOnce('lock coordinator tick failed', error);
789
+ }
790
+ }
791
+ if (tickingCoordinators.size === 0) {
792
+ clearInterval(tickTimer);
793
+ tickTimer = undefined;
794
+ }
795
+ }, TICK_INTERVAL_MS);
796
+ tickTimer.unref?.();
797
+ }
798
+
799
+ /** Stands in until `registerAdmission` supplies the handle's real revoker. */
800
+ function noRevoke() {}
801
+
802
+ /**
803
+ * Fire a revoker in a fire-and-forget context, absorbing both a synchronous throw and a rejected
804
+ * promise. A relayed revoker is `() => Promise<void>` and may reject on a dead sibling port; a
805
+ * discarded rejection would exit the worker under Node's default policy, taking every coordinator on
806
+ * the thread. Callers that must WAIT for the fence (`#revokeAllAndSettle`, `revokeRemoteAdmission`)
807
+ * handle the outcome themselves and do not use this.
808
+ */
809
+ function isPromiseLike(value: unknown): value is Promise<unknown> {
810
+ return value != null && typeof (value as Promise<unknown>).then === 'function';
811
+ }
812
+ function fireRevokeAndForget(revoke: () => void | Promise<void>): void {
813
+ try {
814
+ const outcome = revoke();
815
+ if (isPromiseLike(outcome))
816
+ outcome.catch((error) => warnOnce('a fire-and-forget record lock revoke failed', error));
817
+ } catch (error) {
818
+ warnOnce('a fire-and-forget record lock revoke threw', error);
819
+ }
820
+ }
821
+
822
+ /**
823
+ * Rate-limit rather than latch. These messages report a transport, writer or revoker that failed, and
824
+ * a latch for the life of the process would show an operator the first occurrence and then hide a
825
+ * fault that persists for days. One per message per window is enough to keep a hot loop from flooding
826
+ * the log while still showing that the condition is ongoing.
827
+ */
828
+ const warnedMessages = new Map<string, number>();
829
+ function warnOnce(message: string, detail?: unknown) {
830
+ const now = performance.now();
831
+ const last = warnedMessages.get(message);
832
+ if (last !== undefined && now - last < WARN_INTERVAL_MS) return;
833
+ warnedMessages.set(message, now);
834
+ harperLogger.warn?.(message, detail);
835
+ }
836
+
837
+ export class LockCoordinator {
838
+ readonly database: string;
839
+ readonly table: string;
840
+ readonly nodeId: string;
841
+ readonly transport: ClusterLockTransport;
842
+ #writeControl: (entry: LockControlEntry) => Promise<number | void> | number | void;
843
+ #keyIdOf: (key: any) => unknown;
844
+ #nextTimestamp: () => number;
845
+ #monotonic: () => number;
846
+ #skewMs: number;
847
+ #autoTick: boolean;
848
+ /** Whoever this coordinator's authority was handed to, for replies that land after the swap. */
849
+ #successor: LockCoordinator | undefined;
850
+ /** Set when this coordinator's state was moved to a successor, so `close()` must not expire it. */
851
+ #handedOff = false;
852
+ /**
853
+ * Monotonic reading before which this coordinator may not grant as a home — the §4.3 restart
854
+ * quarantine, and core's own to enforce.
855
+ *
856
+ * A coordinator that started cold has no record of the delegations a previous incarnation issued,
857
+ * and those can still be admitting on their holders. Nothing external bounds them: the home map is
858
+ * immutable, so its generation does not advance merely because a process or a worker restarted. The
859
+ * only instant core can prove nothing else was granting under is this coordinator's own
860
+ * construction, so the quarantine runs `DELEGATION_LEASE_MS + skew` from there — by which point
861
+ * every delegation a previous incarnation could have issued has expired. It costs availability on
862
+ * this node's own share of the ring and nothing elsewhere: keys homed on other nodes are acquired
863
+ * immediately, and `adopt` plus the retirement record waive it wherever a predecessor's authority
864
+ * is actually known.
865
+ *
866
+ * A deployment that can prove a previous incarnation issued nothing overrides it through
867
+ * `ClusterLockTransport.grantableAfterMono`. Generation CHANGES need nothing here: the
868
+ * delegate-side generation check in `#liveDelegation` is what stops the old delegate.
869
+ */
870
+ #grantableAfterMono: number;
871
+ /**
872
+ * How long until this coordinator can rule out authority issued before it took over — 0 when it
873
+ * already can. Non-mutating, unlike `#ownershipHorizon`, which records ownership as a side effect.
874
+ *
875
+ * The waiver is deliberately NOT consulted. `grantableAfterMono` attests that no previous
876
+ * INCARNATION OF THIS PROCESS delegated; it says nothing about a sibling thread that was
877
+ * coordinating until this instant, and a coordinator built while already owning keeps the waiver
878
+ * without ever observing that handoff — which let a takeover worker in a first-incarnation process
879
+ * report a clean drain while the previous owner's delegates were still admitting.
880
+ *
881
+ * The cost is that a freshly built coordinator cannot prove quiescence for a full lease. That falls
882
+ * only on the case that does not need the proof: a node with no delegations yet is bootstrapping
883
+ * generation 1, where there is nothing to drain and no interval to skip.
884
+ */
885
+ unprovenOwnershipMs(): number {
886
+ const horizon = DELEGATION_LEASE_MS + this.#skewMs;
887
+ if (this.#ownedSinceMono === undefined) return horizon;
888
+ return Math.max(0, this.#ownedSinceMono + horizon - this.#monotonic());
889
+ }
890
+ /**
891
+ * When this coordinator was last observed to own coordination, and `undefined` while it does not.
892
+ *
893
+ * The quarantine has to run from here and not only from construction: a coordinator is built when a
894
+ * transport registers, but `ownsCoordination()` can flip to true long afterwards — a thread that
895
+ * took over from an owner that died. By then the construction horizon has aged out, and this
896
+ * coordinator would grant immediately over delegations the previous OWNER issued. Regaining
897
+ * ownership re-arms it for the same reason: something else was coordinating in between.
898
+ */
899
+ #ownedSinceMono: number | undefined;
900
+ /** Rate-limits `tick()`'s ownership poll to `OWNERSHIP_POLL_MS`. */
901
+ #lastOwnershipPollMono = -Infinity;
902
+ /**
903
+ * The `homeIncarnation` this coordinator has been coordinating under. A different one is the
904
+ * transport saying another coordination incarnation ran for this node (§5.1), which is the only
905
+ * statement about CONTINUITY available here — see `#ownershipHorizon`.
906
+ */
907
+ #coordinatingIncarnation: number | undefined;
908
+ /**
909
+ * Set when the caller supplied an explicit horizon. It waives both halves of the quarantine only
910
+ * for this coordinator's FIRST ownership interval, and `#ownershipHorizon` clears it at the first
911
+ * observed gap: the attestation behind it is "no previous incarnation of this process issued
912
+ * anything", which is a claim about process start and not about a sibling thread that coordinated
913
+ * while this one did not.
914
+ */
915
+ #quarantineWaived: boolean;
916
+ /** Keys this node holds a delegation for. */
917
+ #delegations = new Map<unknown, Delegation>();
918
+ /** Grants received but not yet installed because their freshness barrier is still running. */
919
+ #pendingDelegations = new Map<unknown, PendingDelegation>();
920
+ /** Outbound requests whose grant token is not known yet, so an early recall cannot be lost. */
921
+ #pendingRequests = new Map<unknown, PendingRequest>();
922
+ /**
923
+ * Every live admission, by id, and the delegation answerable for it. Coordinator-level rather than
924
+ * per-delegation so a release can find its admission after the delegation was renewed or replaced.
925
+ */
926
+ #admissions = new Map<number, Delegation>();
927
+ #nextAdmissionId = 1;
928
+ /**
929
+ * Admissions this thread obtained from the OWNER thread (harper-pro#852), keyed by a LOCAL id drawn
930
+ * from `#nextAdmissionId` — never the owner's id, so it cannot collide with a live local admission.
931
+ * This thread holds the handle (native key, staged writes, lease timer) while the delegation that
932
+ * authorizes it lives on the owner. Allocated lazily so a coordinator that never serves an off-owner
933
+ * lock pays nothing. An entry is kept after the caller unlocks, revoker included, until the handle's
934
+ * own lease runs out (owner-side §6 retention: an unlocked-but-staged write is still fenceable).
935
+ */
936
+ #remoteAdmissions: Map<number, RemoteAdmission> | undefined;
937
+ /** Owner admission id → this thread's local id, so an inbound revoke (which names the owner id)
938
+ * reaches the right entry. */
939
+ #remoteByOwnerId: Map<number, number> | undefined;
940
+ /** Owner admission ids whose revoke arrived before `#acquireFromOwner` installed the entry, with the
941
+ * ack resolvers waiting on the eventual fence and the monotonic time they arrived. Drained when the
942
+ * entry installs (`#acquireFromOwner`); an entry whose acquire never lands (a revoke for an admission
943
+ * this thread already dropped) is resolved and swept once its wait exceeds a lease (`tick`). */
944
+ #pendingRemoteRevokes: Map<number, { resolvers: (() => void)[]; at: number }> | undefined;
945
+ /**
946
+ * Bumped every time every relayed admission is fenced wholesale (`fenceAllRemoteAdmissions`, i.e. the
947
+ * owner worker is gone). `#acquireFromOwner` samples it before it asks the owner and re-checks after
948
+ * the reply: a grant minted by an owner that has since been declared gone must NOT be installed, or
949
+ * this thread would start writing under an admission the replacement owner knows nothing about while
950
+ * that owner, starting empty, grants the same key to somebody else. Carried forward on handoff so a
951
+ * transport swap mid-acquire cannot reset it and re-open the window.
952
+ */
953
+ #relayGeneration = 0;
954
+ #relayedAdmissions = 0;
955
+ /** Keys this node homes, and who holds each one. */
956
+ #grants = new Map<unknown, HomeGrant>();
957
+ /** Per-requester counts, so one peer cannot fill the home's table on its own. */
958
+ #grantsByRequester = new Map<string, number>();
959
+ /** Clean-handoff lineage outlives grants and is retained independently under its own cap. */
960
+ #dependencySets = new Map<unknown, LockDependencySet>();
961
+ #everDelegated = new DelegatedKeyFilter();
962
+ #freshnessGeneration: number | undefined;
963
+ /** False after any interval whose delegation history this coordinator could not have observed. */
964
+ #trustVirginKeys: boolean;
965
+ #counter = 0;
966
+ /**
967
+ * Closed coordinators must not keep admitting. `close()` sets this AND expires every delegation,
968
+ * because a handle already handed out checks only its own lease — clearing the table alone would
969
+ * let a successor coordinator (a re-registered transport) grant the same key with no lease time
970
+ * elapsed.
971
+ */
972
+ #closed = false;
973
+ // A latched warning would make a permanently misrouted deployment look like a quiet cluster.
974
+ #droppedOffOwner = 0;
975
+ #lastOffOwnerWarn = 0;
976
+
977
+ constructor(options: LockCoordinatorOptions) {
978
+ if (!isNodeName(options.nodeId) || NON_DISTINCTIVE_NODE_NAMES.has(options.nodeId))
979
+ throw new LockUnavailableError(
980
+ `Cluster record locks need a distinctive node name to derive a key's home from, but this node identifies as "${options.nodeId}". Set node.hostname to this node's name in system.hdb_nodes.`
981
+ );
982
+ this.database = options.database;
983
+ this.table = options.table;
984
+ liveCoordinators.add(this);
985
+ this.nodeId = options.nodeId;
986
+ this.transport = options.transport;
987
+ this.#writeControl = options.writeControl;
988
+ this.#keyIdOf = options.keyIdOf;
989
+ this.#nextTimestamp = options.nextTimestamp;
990
+ this.#monotonic = options.monotonic ?? (() => performance.now());
991
+ this.#skewMs = options.skewMs ?? LOCK_LEASE_SKEW_MS;
992
+ this.#autoTick = options.autoTick !== false;
993
+ // Anchored at construction, which is the only instant core can prove nothing else was granting
994
+ // under: process start is wrong (`performance.now()` is process-wide inside a worker too, so a
995
+ // replacement coordinating thread would read an uptime far past it) and so is thread start (a
996
+ // thread can take coordination ownership long after it booted). `adopt` and the retirement
997
+ // record below waive or carry the horizon wherever a predecessor's authority is actually known.
998
+ this.#quarantineWaived = options.grantableAfterMono !== undefined;
999
+ this.#trustVirginKeys = this.#quarantineWaived;
1000
+ // Ownership observed here, not lazily on the first grant: a coordinator built while this thread
1001
+ // already coordinates has owned it since construction, and the construction horizon covers that.
1002
+ // Leaving it unset until a grant would date ownership from the grant and re-quarantine a reload.
1003
+ try {
1004
+ if (options.transport.ownsCoordination()) {
1005
+ this.#ownedSinceMono = this.#monotonic();
1006
+ // The incarnation it has been coordinating under, so the first grant does not read its own
1007
+ // construction as a takeover. Undefined when there is no map yet, which `#ownershipHorizon`
1008
+ // treats as a takeover — the conservative direction.
1009
+ this.#coordinatingIncarnation = options.transport.homeMap(options.database)?.homeIncarnation;
1010
+ }
1011
+ } catch {
1012
+ // A transport that cannot answer yet is not owning yet; `#ownershipHorizon` will observe it.
1013
+ }
1014
+ this.#grantableAfterMono = options.grantableAfterMono ?? this.#monotonic() + DELEGATION_LEASE_MS + this.#skewMs;
1015
+ options.adopt?.handOffTo(this);
1016
+ const retired = retiredCoordinators.get(this.#retirementKey());
1017
+ if (retired) {
1018
+ this.#counter = Math.max(this.#counter, retired.counter);
1019
+ this.#grantableAfterMono = Math.max(this.#grantableAfterMono, retired.grantableAfterMono);
1020
+ // A close without a successor discarded the key filter and retained dependency sets. An
1021
+ // explicit cold-start waiver on the replacement says nothing about that discarded history.
1022
+ this.#trustVirginKeys = false;
1023
+ }
1024
+ }
1025
+
1026
+ /**
1027
+ * Move this coordinator's live authority to the coordinator replacing it. Called from the
1028
+ * successor's constructor, before `close()`, so nothing is dropped in between. Both coordinators
1029
+ * run in the same thread for the same node, so the delegations and grants are still this node's —
1030
+ * only the transport object underneath them changed.
1031
+ */
1032
+ handOffTo(successor: LockCoordinator): void {
1033
+ if (this.#handedOff) return;
1034
+ this.#handedOff = true;
1035
+ // Kept so an acquisition still awaiting a home's reply can install it on whoever holds this
1036
+ // node's authority when the reply lands, rather than on a coordinator nothing consults.
1037
+ this.#successor = successor;
1038
+ for (const [keyId, delegation] of this.#delegations) successor.#delegations.set(keyId, delegation);
1039
+ for (const [keyId, pending] of this.#pendingDelegations) successor.#pendingDelegations.set(keyId, pending);
1040
+ for (const [keyId, request] of this.#pendingRequests) successor.#pendingRequests.set(keyId, request);
1041
+ for (const [keyId, grant] of this.#grants) successor.#grants.set(keyId, grant);
1042
+ for (const [requester, count] of this.#grantsByRequester) successor.#grantsByRequester.set(requester, count);
1043
+ // The admission index moves with the delegations it points into. Without it a handle admitted
1044
+ // on the predecessor could never be released through the successor — `release` addresses the
1045
+ // admission, so the entry would sit on the delegation forever and no recall could drain it.
1046
+ for (const [admissionId, delegation] of this.#admissions) successor.#admissions.set(admissionId, delegation);
1047
+ // Remote admissions (harper-pro#852) move with their handles: this thread still holds the native
1048
+ // key and the staged write for each, so the successor must be the one a later `release` or a
1049
+ // relayed `revoke` reaches. Their local ids stay valid (the successor's own `#nextAdmissionId` is
1050
+ // carried forward below), and the owner-id index and any not-yet-installed revokes move with them.
1051
+ if (this.#remoteAdmissions) {
1052
+ const into = (successor.#remoteAdmissions ??= new Map());
1053
+ for (const [localId, remote] of this.#remoteAdmissions) into.set(localId, remote);
1054
+ const ownerIndex = (successor.#remoteByOwnerId ??= new Map());
1055
+ for (const [ownerId, localId] of this.#remoteByOwnerId ?? []) ownerIndex.set(ownerId, localId);
1056
+ this.#remoteAdmissions = undefined;
1057
+ this.#remoteByOwnerId = undefined;
1058
+ }
1059
+ if (this.#pendingRemoteRevokes) {
1060
+ const into = (successor.#pendingRemoteRevokes ??= new Map());
1061
+ for (const [ownerId, entry] of this.#pendingRemoteRevokes) {
1062
+ const existing = into.get(ownerId);
1063
+ if (existing) existing.resolvers.push(...entry.resolvers);
1064
+ else into.set(ownerId, entry);
1065
+ }
1066
+ this.#pendingRemoteRevokes = undefined;
1067
+ }
1068
+ successor.#relayedAdmissions += this.#relayedAdmissions;
1069
+ // Never backwards: an acquire that sampled the predecessor must still see a bump the successor
1070
+ // (or the predecessor) already recorded, so a swap mid-acquire cannot re-open the orphan window.
1071
+ successor.#relayGeneration = Math.max(successor.#relayGeneration, this.#relayGeneration);
1072
+ // Neither counter may restart. A repeated token would compare equal to one the predecessor
1073
+ // already issued for a different delegation; a repeated admission id would address the wrong
1074
+ // admission in the map just carried over.
1075
+ successor.#counter = Math.max(successor.#counter, this.#counter);
1076
+ successor.#nextAdmissionId = Math.max(successor.#nextAdmissionId, this.#nextAdmissionId);
1077
+ // The predecessor's horizon EXACTLY, not the successor's freshly computed one. Adoption means the
1078
+ // successor now knows everything the predecessor knew, so it faces the same cold-start hazard and
1079
+ // no more: recomputing from its own construction would quarantine a node for a full delegation
1080
+ // lease on every transport reload, and clearing it would let a swap inside the window grant over
1081
+ // an unseen predecessor incarnation.
1082
+ successor.#grantableAfterMono = this.#grantableAfterMono;
1083
+ // And the ownership clock: a transport reload does not change which thread coordinates, so the
1084
+ // successor inherits how long this one has owned it rather than starting a fresh interval. The
1085
+ // waiver rides along with it and in the same direction: the successor reads `grantableAfterMono`
1086
+ // off the new transport and would otherwise re-waive a horizon this coordinator had already lost
1087
+ // to a takeover.
1088
+ //
1089
+ // Only what this coordinator actually OBSERVED may override that, though. `undefined` on either
1090
+ // of these means "never saw it" — a coordinator built on a non-owning thread, or before a map
1091
+ // was available — and the successor has just read the new transport, which is not less current.
1092
+ // Carrying the blank over it re-armed a quarantine on a node that had never stopped coordinating.
1093
+ if (this.#ownedSinceMono !== undefined) {
1094
+ successor.#ownedSinceMono = this.#ownedSinceMono;
1095
+ successor.#quarantineWaived = this.#quarantineWaived;
1096
+ }
1097
+ if (this.#coordinatingIncarnation !== undefined) successor.#coordinatingIncarnation = this.#coordinatingIncarnation;
1098
+ successor.#dependencySets = this.#dependencySets;
1099
+ successor.#everDelegated.copyFrom(this.#everDelegated);
1100
+ successor.#freshnessGeneration = this.#freshnessGeneration;
1101
+ successor.#trustVirginKeys = this.#trustVirginKeys;
1102
+ this.#dependencySets = new Map();
1103
+ this.#everDelegated.clear();
1104
+ this.#delegations.clear();
1105
+ this.#pendingDelegations.clear();
1106
+ this.#pendingRequests.clear();
1107
+ this.#grants.clear();
1108
+ this.#grantsByRequester.clear();
1109
+ this.#admissions.clear();
1110
+ if (
1111
+ successor.#delegations.size > 0 ||
1112
+ successor.#pendingDelegations.size > 0 ||
1113
+ successor.#grants.size > 0 ||
1114
+ (successor.#remoteAdmissions?.size ?? 0) > 0 ||
1115
+ (successor.#pendingRemoteRevokes?.size ?? 0) > 0
1116
+ )
1117
+ successor.#startTicking();
1118
+ }
1119
+
1120
+ /**
1121
+ * For `cluster_status`: delegations held, delegations issued, live admissions, admissions still
1122
+ * revocable, misrouted calls. `revocable` counts the ones an unlock left fenceable as well, so the
1123
+ * retention `#pruneAdmissions` bounds is visible rather than inferred from `admitted`.
1124
+ */
1125
+ get stats(): {
1126
+ delegations: number;
1127
+ granted: number;
1128
+ admitted: number;
1129
+ revocable: number;
1130
+ droppedOffOwner: number;
1131
+ relayedAdmissions: number;
1132
+ } {
1133
+ let admitted = 0;
1134
+ let revocable = 0;
1135
+ for (const delegation of this.#delegations.values()) {
1136
+ admitted += delegation.holding;
1137
+ revocable += delegation.admissions.size;
1138
+ }
1139
+ return {
1140
+ delegations: this.#delegations.size,
1141
+ granted: this.#grants.size,
1142
+ admitted,
1143
+ revocable,
1144
+ droppedOffOwner: this.#droppedOffOwner,
1145
+ // Admissions this thread has obtained from the owner worker (harper-pro#852), cumulative
1146
+ // rather than current: what it makes visible in `cluster_status` is that off-owner `lock()`s
1147
+ // are being served here at all.
1148
+ relayedAdmissions: this.#relayedAdmissions,
1149
+ };
1150
+ }
1151
+
1152
+ /**
1153
+ * Admit a critical section for a key whose native lock this thread already holds. Resolves with
1154
+ * the stamp and the monotonic reading the lease runs from; rejects 423 when no delegation could be
1155
+ * obtained in time, or 503 when the guarantee cannot be established at all.
1156
+ *
1157
+ * The amortization is the first branch: a live, un-recalled delegation with enough time left costs
1158
+ * zero cluster messages.
1159
+ */
1160
+ async acquire(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
1161
+ // `Table.lock()` captures a coordinator and only reaches here after the native key lock, which
1162
+ // can wait the caller's whole timeout — long enough for a transport swap to close what it
1163
+ // captured. Authority moved to the successor rather than away, so run there instead of
1164
+ // rejecting a caller that is already holding the key.
1165
+ const authority = this.#authority();
1166
+ if (authority !== this) return authority.acquire(key, leaseMs, waitMs);
1167
+ if (this.#closed) throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1168
+ if (!this.transport.ownsCoordination()) {
1169
+ // Off the coordinating thread. A transport that can relay obtains the admission from the
1170
+ // owner and installs the handle's revoker here (harper-pro#852); one that cannot fails
1171
+ // closed, the historical contract.
1172
+ // BOTH halves or neither. A transport that could acquire but not release would route locks and
1173
+ // then never forward an unlock, leaving every admission `holding` on the owner for its full
1174
+ // lease and stalling peer recalls — worse than the honest 503 below.
1175
+ if (this.transport.acquireOnOwner && this.transport.releaseOnOwner)
1176
+ return this.#acquireFromOwner(key, leaseMs, waitMs);
1177
+ throw new LockUnavailableError(
1178
+ 'Cluster record lock coordination is not owned by this worker thread; retry so the request reaches the coordinating thread'
1179
+ );
1180
+ }
1181
+ const keyId = this.#keyIdOf(key);
1182
+ const deadlineMono = this.#monotonic() + waitMs;
1183
+
1184
+ acquisition: for (;;) {
1185
+ const homeMap = this.transport.homeMap(this.database);
1186
+ // No agreed map means no agreed ring, and a ring guessed from whoever looks reachable is
1187
+ // exactly the asymmetric-partition failure a single arbiter exists to remove.
1188
+ if (!homeMap || !this.#generationIsCurrent(homeMap.generation))
1189
+ throw new LockUnavailableError(
1190
+ `No agreed record lock home map for ${this.database}; cluster record locks are unavailable until one is established`
1191
+ );
1192
+ const delegation = this.#liveDelegation(keyId, leaseMs, homeMap.generation);
1193
+ if (delegation) return this.#admit(delegation, leaseMs);
1194
+
1195
+ const home = homeFor(this.#ringKey(keyId), homeMap.homes);
1196
+ if (!home)
1197
+ throw new LockUnavailableError(`The record lock home map for ${this.database} names no nodes to home a key on`);
1198
+
1199
+ // Anchored before the send: the home starts its own clock when it grants, so measuring the
1200
+ // delegation from the reply's arrival would hand a delayed reply more time than the home is
1201
+ // holding the key for.
1202
+ const requestedAtMono = this.#monotonic();
1203
+ const pendingRequest: PendingRequest = {};
1204
+ this.#pendingRequests.set(keyId, pendingRequest);
1205
+ let reply: DelegationReply;
1206
+ try {
1207
+ reply =
1208
+ home === this.nodeId
1209
+ ? this.#grantLocally(keyId, key, homeMap, leaseMs)
1210
+ : await this.#requestRemotely(home, keyId, key, homeMap, leaseMs, deadlineMono);
1211
+ } catch (error) {
1212
+ const requestAuthority = this.#authority();
1213
+ if (requestAuthority.#pendingRequests.get(keyId) === pendingRequest)
1214
+ requestAuthority.#pendingRequests.delete(keyId);
1215
+ throw error;
1216
+ }
1217
+
1218
+ // A transport swap can land while a request is in flight. The grant is authority for this
1219
+ // NODE, and the successor is this node now — installing it here would leave a delegation
1220
+ // nothing consults, admitting a caller that no recall can reach.
1221
+ let authority = this.#authority();
1222
+ if (authority.#pendingRequests.get(keyId) === pendingRequest) authority.#pendingRequests.delete(keyId);
1223
+ if (authority.#closed)
1224
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1225
+
1226
+ const replyDependencies =
1227
+ reply.dependencies === null ? null : normalizeDependencies(reply.dependencies, homeMap.homes);
1228
+ if (
1229
+ reply.granted &&
1230
+ isFencingToken(reply.token) &&
1231
+ isDuration(reply.leaseMs, MIN_LOCK_LEASE_MS, DELEGATION_LEASE_MS) &&
1232
+ replyDependencies !== undefined
1233
+ ) {
1234
+ const recalledBeforeReply =
1235
+ pendingRequest.recalledToken !== undefined && compareTokens(pendingRequest.recalledToken, reply.token) === 0;
1236
+ const existing = authority.#delegations.get(keyId);
1237
+ if (
1238
+ existing &&
1239
+ !existing.recalled &&
1240
+ existing.token[0] === reply.token[0] &&
1241
+ compareTokens(existing.token, reply.token) < 0
1242
+ ) {
1243
+ // This is a continuous renewal of authority this node already made fresh. No other
1244
+ // delegate could have held the key, so repeating the barrier — especially recovery —
1245
+ // buys no freshness. The home still returns its requirement for the case where the
1246
+ // requester lost local state and has no existing delegation to prove continuity.
1247
+ const renewed = authority.#installDelegation(
1248
+ keyId,
1249
+ key,
1250
+ reply.token,
1251
+ reply.leaseMs,
1252
+ requestedAtMono,
1253
+ existing.dependencies
1254
+ );
1255
+ if (renewed && recalledBeforeReply) {
1256
+ // The renewed delegation inherits every admission from the old token. Drain those
1257
+ // admissions through the ordinary recall path; surrendering directly would revoke
1258
+ // their handles and clear the home's grant while their critical sections still run.
1259
+ await authority.onDelegationRecall({ key, token: reply.token });
1260
+ if (authority.#closed)
1261
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1262
+ continue acquisition;
1263
+ }
1264
+ if (renewed && renewed.expiresMono - authority.#monotonic() >= leaseMs)
1265
+ return authority.#admit(renewed, leaseMs);
1266
+ continue acquisition;
1267
+ }
1268
+ const pending = createPendingDelegation(key, reply.token, recalledBeforeReply);
1269
+ authority.#pendingDelegations.set(keyId, pending);
1270
+ let dependencies: LockDependencySet;
1271
+ try {
1272
+ let requirement = replyDependencies;
1273
+ for (;;) {
1274
+ const barrierAuthority = authority;
1275
+ dependencies = await barrierAuthority.#establishFreshness(
1276
+ key,
1277
+ requirement,
1278
+ homeMap.homes,
1279
+ deadlineMono,
1280
+ pending
1281
+ );
1282
+ authority = this.#authority();
1283
+ if (authority === barrierAuthority) break;
1284
+ requirement = dependencies;
1285
+ }
1286
+ } catch (error) {
1287
+ authority = this.#authority();
1288
+ if (authority.#pendingDelegations.get(keyId) === pending) authority.#pendingDelegations.delete(keyId);
1289
+ await authority.#releaseUnclaimedGrant(key, reply.token);
1290
+ if (error === PENDING_DELEGATION_RECALLED) {
1291
+ if (authority.#closed)
1292
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1293
+ continue acquisition;
1294
+ }
1295
+ throw new LockUnavailableError(
1296
+ `Could not establish successor freshness for ${this.database}.${this.table}: ${(error as Error)?.message ?? error}`
1297
+ );
1298
+ }
1299
+ const currentPending = authority.#pendingDelegations.get(keyId);
1300
+ if (currentPending === pending) authority.#pendingDelegations.delete(keyId);
1301
+ if (currentPending !== pending || pending.recalled) {
1302
+ await authority.#releaseUnclaimedGrant(key, reply.token);
1303
+ if (authority.#closed)
1304
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1305
+ continue;
1306
+ }
1307
+ // The generation can advance across the await. A grant minted under a superseded one is
1308
+ // authority for a ring that no longer exists: the key may be homed elsewhere now, and that
1309
+ // home can already have granted it to another node. `#liveDelegation` rejects a stale
1310
+ // token on the NEXT pass, which is too late — this pass would have admitted on it first.
1311
+ if (reply.token[0] !== authority.transport.homeMap(this.database)?.generation) {
1312
+ // Hand it back rather than let the old home hold a key nobody is using for a full lease.
1313
+ authority.#releaseUnclaimedGrant(key, reply.token);
1314
+ } else {
1315
+ const installed = authority.#installDelegation(
1316
+ keyId,
1317
+ key,
1318
+ reply.token,
1319
+ reply.leaseMs,
1320
+ requestedAtMono,
1321
+ dependencies
1322
+ );
1323
+ // A reply that outlived its own delegation grants nothing; fall through and ask again
1324
+ // rather than admitting on authority the home has already expired.
1325
+ if (installed && !installed.recalled && installed.expiresMono - authority.#monotonic() >= leaseMs)
1326
+ return authority.#admit(installed, leaseMs);
1327
+ // A long barrier can consume enough of the delegation that the requested lease no longer
1328
+ // fits. Surrender the unused installed delegation before retrying; otherwise the home renews
1329
+ // it in place on every pass and contenders wait for its full deadline.
1330
+ if (installed && compareTokens(installed.token, reply.token) === 0 && installed.holding === 0)
1331
+ await authority.#surrender(keyId, installed);
1332
+ else if (!installed) await authority.#releaseUnclaimedGrant(key, reply.token);
1333
+ }
1334
+ } else if (reply.granted) {
1335
+ if (isFencingToken(reply.token)) authority.#releaseUnclaimedGrant(key, reply.token);
1336
+ throw new LockUnavailableError(
1337
+ `The home node for this key on ${this.database}.${this.table} returned a delegation with no usable token or freshness requirement`
1338
+ );
1339
+ }
1340
+ if (reply.reason === 'capacity')
1341
+ throw new LockUnavailableError(`Too many record lock delegations in flight on ${this.database}`);
1342
+ if (reply.reason === 'quarantine')
1343
+ throw new LockUnavailableError(
1344
+ `The home node for this key on ${this.database}.${this.table} restarted and cannot grant until the delegations its previous incarnation issued have expired`
1345
+ );
1346
+ // Neither of these can be waited out inside a `lock()` timeout, and retrying them spends the
1347
+ // caller's whole budget holding the native key only to answer 423 for a key nobody holds.
1348
+ if (reply.reason === 'generation')
1349
+ throw new LockUnavailableError(
1350
+ `The home node for this key on ${this.database}.${this.table} holds record lock home map generation ${reply.generation ?? 'unknown'} and this node holds ${homeMap.generation}; cluster record locks are unavailable until they agree`
1351
+ );
1352
+ if (reply.reason === 'unknown-node')
1353
+ throw new LockUnavailableError(
1354
+ `This node is not named in the record lock home map for ${this.database}, so it can neither home a key nor take a cluster lock on one`
1355
+ );
1356
+ if (reply.reason === 'not-home')
1357
+ // The home disagrees about the ring. Re-reading the map on the next pass is the fix, and it
1358
+ // converges if our copy is the stale one — so unlike the denials above this one is worth
1359
+ // retrying. If it is NOT stale (two maps under one generation number) it never converges,
1360
+ // which the terminal answer below is what handles.
1361
+ warnOnce('record lock home disagreed about the ring', { database: this.database, table: this.table });
1362
+
1363
+ const remaining = deadlineMono - this.#monotonic();
1364
+ if (remaining <= 0) {
1365
+ // 423 says "someone else holds this key", so only a denial that actually means that may end
1366
+ // as one. Every other reason ran out the clock without the key ever being held, and reporting
1367
+ // contention for it sends the caller to retry a condition no timeout can outlast.
1368
+ if (reply.reason === 'contended') throw new ClientError('Record is locked and was not released in time', 423);
1369
+ throw new LockUnavailableError(
1370
+ `Could not establish a cluster record lock on ${this.database}.${this.table} within the wait: the key's home answered ${reply.reason ?? (reply.granted ? 'grants that arrived too late to use' : 'nothing usable')}`
1371
+ );
1372
+ }
1373
+ await delay(Math.min(reply.retryAfterMs ?? 25, remaining)).promise;
1374
+ if (this.#closed) {
1375
+ // Same swap, landing in the backoff instead. Carry the remaining wait so the deadline the
1376
+ // caller asked for is preserved across the hop.
1377
+ const successor = this.#authority();
1378
+ if (successor === this)
1379
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1380
+ return successor.acquire(key, leaseMs, Math.max(0, deadlineMono - this.#monotonic()));
1381
+ }
1382
+ }
1383
+ }
1384
+
1385
+ /**
1386
+ * Obtain an admission from the owner worker for a `lock()` served on a non-owner thread
1387
+ * (harper-pro#852), recorded under a LOCAL id so `registerAdmission`/`release` can never collide it
1388
+ * with a live local admission. `acquireOnOwner` is bounded by the transport, inside the caller's
1389
+ * `waitMs`; the race here is core's own backstop against a transport that never answers, so `lock()`
1390
+ * cannot hang far past the wait it was given.
1391
+ */
1392
+ async #acquireFromOwner(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
1393
+ // The transport bounds this wait (harper-pro's acquire has its own `waitMs`-scaled timeout); the
1394
+ // core-side race is a last-resort backstop set strictly beyond that bound — the transport is asked
1395
+ // for `waitMs - hop` and core's deadline is `waitMs + REMOTE_ACQUIRE_BACKSTOP_MS` — so it fires only
1396
+ // if the transport never returns, never ahead of the transport's own timeout. Without it a wedged
1397
+ // owner would hang `lock()` past its `waitMs`.
1398
+ // Sampled BEFORE the request goes out, on the authority that will install the result.
1399
+ const startGeneration = this.#authority().#relayGeneration;
1400
+ // The hop allowance comes OUT of the caller's budget, never on top of it: `lock()` holds the native
1401
+ // key for this whole wait, so overshooting `waitMs` blocks every other worker on the key for the
1402
+ // overshoot. The transport is asked for the reduced wait and answers within the caller's budget;
1403
+ // core's deadline is the caller's `waitMs` plus a small net that fires only if the transport never
1404
+ // answers at all.
1405
+ const hop = Math.min(REMOTE_ACQUIRE_HOP_MS, Math.floor(waitMs / 2));
1406
+ // A transport that throws SYNCHRONOUSLY (a sibling port already gone) must still reach the
1407
+ // normalization below: escaping raw gives the caller a 500 where it needs the retryable 503.
1408
+ // `#beginRecall` wraps its transport call for the same reason.
1409
+ const acquire = Promise.resolve().then(() =>
1410
+ this.transport.acquireOnOwner!(this.database, this.table, key, leaseMs, waitMs - hop)
1411
+ );
1412
+ let backstopWon = false;
1413
+ // If the backstop wins the race, the owner may still grant afterward: release that grant back so it
1414
+ // does not sit `holding` on the owner for its whole lease. (The transport releases a late reply too;
1415
+ // this closes the case where core's backstop fired first.)
1416
+ acquire.then(
1417
+ (round) => {
1418
+ if (backstopWon) this.#authority().#releaseOnOwnerSafely(key, round.admissionId);
1419
+ },
1420
+ () => {}
1421
+ );
1422
+ const round = await withDeadline(
1423
+ acquire,
1424
+ waitMs + REMOTE_ACQUIRE_BACKSTOP_MS,
1425
+ 'the coordinating worker did not answer'
1426
+ ).catch((error) => {
1427
+ backstopWon = true;
1428
+ throw error instanceof LockUnavailableError
1429
+ ? error
1430
+ : new LockUnavailableError(
1431
+ `Could not obtain a cluster record lock on ${this.database}.${this.table} from the coordinating worker: ${(error as Error)?.message ?? error}`
1432
+ );
1433
+ });
1434
+ const authority = this.#authority();
1435
+ if (authority.#closed) {
1436
+ // The coordinator closed while the owner was granting. The owner still holds this admission;
1437
+ // hand it straight back rather than leaving it outstanding for its whole lease.
1438
+ authority.#releaseOnOwnerSafely(key, round.admissionId);
1439
+ throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
1440
+ }
1441
+ if (authority.#relayGeneration !== startGeneration) {
1442
+ // Every relayed admission was fenced while this grant was in flight: the owner that minted it
1443
+ // has been declared gone, and the replacement starts with no record of it. Installing it now
1444
+ // would let this thread write under an admission the new owner can neither see nor recall,
1445
+ // while that owner grants the same key to another worker — two writers. Fail closed and hand
1446
+ // the grant back; the caller retries against the new owner.
1447
+ authority.#releaseOnOwnerSafely(key, round.admissionId);
1448
+ throw new LockUnavailableError(
1449
+ 'The record lock coordinating worker changed while this lock was being granted; retry against the new owner'
1450
+ );
1451
+ }
1452
+ // A live entry already keyed to this owner id means the owner reused an id its counter had already
1453
+ // issued — only possible if a replacement owner restarted its sequence. The old entry is from that
1454
+ // prior owner incarnation and its delegation is gone: fence it fail-closed before the new one takes
1455
+ // the id, so a later revoke for this id can never address the stale handle.
1456
+ const collidingLocalId = authority.#remoteByOwnerId?.get(round.admissionId);
1457
+ if (collidingLocalId !== undefined) {
1458
+ const stale = authority.#remoteAdmissions?.get(collidingLocalId);
1459
+ if (stale) {
1460
+ fireRevokeAndForget(stale.revoke);
1461
+ authority.#dropRemoteAdmission(collidingLocalId, stale);
1462
+ }
1463
+ }
1464
+ const localId = authority.#nextAdmissionId++;
1465
+ const entry: RemoteAdmission = {
1466
+ ownerAdmissionId: round.admissionId,
1467
+ key,
1468
+ revoke: noRevoke,
1469
+ expiresMono: round.mintedMono + leaseMs,
1470
+ revoked: false,
1471
+ fenceWaiters: [],
1472
+ };
1473
+ (authority.#remoteAdmissions ??= new Map()).set(localId, entry);
1474
+ (authority.#remoteByOwnerId ??= new Map()).set(round.admissionId, localId);
1475
+ // A revoke that raced ahead of this install (the owner registered its revoker before it posted
1476
+ // the grant reply, so a recall in that window fired first): apply it now so the ack the owner is
1477
+ // waiting on cannot resolve before the handle is fenced.
1478
+ const pending = authority.#pendingRemoteRevokes?.get(round.admissionId);
1479
+ if (pending) {
1480
+ authority.#pendingRemoteRevokes!.delete(round.admissionId);
1481
+ entry.revoked = true;
1482
+ entry.fenceWaiters.push(...pending.resolvers);
1483
+ }
1484
+ authority.#relayedAdmissions++;
1485
+ // Tick so an entry whose caller never releases (its lease elapses instead) is still pruned.
1486
+ authority.#startTicking();
1487
+ // The LOCAL id is what the handle registers and releases against; the round the owner minted
1488
+ // still carries its own id, which only the owner-facing release/revoke messages use.
1489
+ return { tsR: round.tsR, mintedMono: round.mintedMono, admissionId: localId };
1490
+ }
1491
+
1492
+ /**
1493
+ * Fence a remote admission's handle on this thread, driven by a recall or surrender on the owner
1494
+ * (harper-pro#852), addressed by the OWNER's admission id. Resolves once the handle is provably
1495
+ * fenced — its `revokeLease` has run — so the owner's ack cannot claim a fence the handle has not
1496
+ * taken; the owner waits on this before writing the release. A revoke that arrives before the handle
1497
+ * is installed, or before `registerAdmission` supplies the real revoker, resolves only when the fence
1498
+ * finally lands. A revoker that throws, or whose promise rejects, fails this promise rather than
1499
+ * resolving it, so the owner falls back to its own lease bound rather than being told the fence
1500
+ * succeeded.
1501
+ */
1502
+ revokeRemoteAdmission(ownerAdmissionId: number): Promise<void> {
1503
+ const localId = this.#remoteByOwnerId?.get(ownerAdmissionId);
1504
+ if (localId === undefined) {
1505
+ // The entry is not installed yet (a revoke that raced ahead of the grant reply): latch the ack
1506
+ // resolver so `#acquireFromOwner` can carry it, and never fence-then-resolve a handle that does
1507
+ // not exist. If the acquire never lands (a revoke for an admission already dropped), `tick`
1508
+ // resolves and sweeps it after a lease so the resolver cannot leak.
1509
+ return new Promise<void>((resolve) => {
1510
+ const map = (this.#pendingRemoteRevokes ??= new Map());
1511
+ const pending = map.get(ownerAdmissionId);
1512
+ if (pending) pending.resolvers.push(resolve);
1513
+ else map.set(ownerAdmissionId, { resolvers: [resolve], at: this.#monotonic() });
1514
+ this.#startTicking();
1515
+ });
1516
+ }
1517
+ const remote = this.#remoteAdmissions?.get(localId);
1518
+ if (!remote) return Promise.resolve();
1519
+ remote.revoked = true;
1520
+ if (remote.revoke === noRevoke) {
1521
+ // The handle has not registered its revoker yet; resolve when `registerAdmission` fences it.
1522
+ return new Promise<void>((resolve) => remote.fenceWaiters.push(resolve));
1523
+ }
1524
+ return Promise.resolve(this.#fenceRemoteAdmission(localId, remote));
1525
+ }
1526
+
1527
+ /**
1528
+ * Fire a remote admission's revoker and report when the handle is PROVABLY fenced. `registerAdmission`
1529
+ * accepts an async revoker, so an outcome that is promise-like is not a fence until it fulfils: the
1530
+ * entry is dropped — and any ack waiting on it settled — only then. A rejection is not a fence at all,
1531
+ * so it propagates and the entry is LEFT for the lease sweep: a retry can fire the revoker again, and
1532
+ * the owner either sees the failure or waits its own lease bound out rather than writing the release
1533
+ * and admitting a successor over a live writer. Unlike a plain release, which retains the entry
1534
+ * because a staged write is still committable until the lease.
1535
+ */
1536
+ #fenceRemoteAdmission(localId: number, remote: RemoteAdmission): void | Promise<void> {
1537
+ let fenced: void | Promise<void>;
1538
+ try {
1539
+ fenced = remote.revoke();
1540
+ } catch (error) {
1541
+ // A synchronous throw is not a fence either, and it must not escape past the promise this
1542
+ // function's callers hand to the owner: surface it as a rejection instead.
1543
+ warnOnce('a relayed record lock fence failed; the handle lease still bounds it', error);
1544
+ return Promise.reject(error);
1545
+ }
1546
+ if (!isPromiseLike(fenced)) {
1547
+ this.#dropRemoteAdmission(localId, remote);
1548
+ return;
1549
+ }
1550
+ return fenced.then(
1551
+ () => {
1552
+ this.#dropRemoteAdmission(localId, remote);
1553
+ },
1554
+ (error) => {
1555
+ warnOnce('a relayed record lock fence failed; the handle lease still bounds it', error);
1556
+ throw error;
1557
+ }
1558
+ );
1559
+ }
1560
+
1561
+ /**
1562
+ * Fence every relayed handle this thread holds, fail-closed (harper-pro#852). Called when the owner
1563
+ * worker that granted them is gone (its thread exited, so its delegation table died with it): the
1564
+ * handles are no longer backed by any delegation, so nothing here may keep committing. There is no
1565
+ * owner left to tell, so this only revokes locally and drops the entries; a still-pending ack is
1566
+ * resolved, since the handle can no longer commit.
1567
+ */
1568
+ fenceAllRemoteAdmissions(): void {
1569
+ // Before anything is torn down, so an acquire already in flight to the departed owner fails its
1570
+ // post-reply generation check rather than installing an orphaned admission.
1571
+ this.#relayGeneration++;
1572
+ if (this.#remoteAdmissions) {
1573
+ for (const remote of this.#remoteAdmissions.values()) {
1574
+ fireRevokeAndForget(remote.revoke);
1575
+ this.#settleFenceWaiters(remote);
1576
+ }
1577
+ this.#remoteAdmissions = undefined;
1578
+ this.#remoteByOwnerId = undefined;
1579
+ }
1580
+ if (this.#pendingRemoteRevokes) {
1581
+ for (const entry of this.#pendingRemoteRevokes.values()) for (const resolve of entry.resolvers) resolve();
1582
+ this.#pendingRemoteRevokes = undefined;
1583
+ }
1584
+ }
1585
+
1586
+ /**
1587
+ * Forward a release to the owner, naming the admission by the OWNER's id. Core deliberately carries no
1588
+ * owner epoch of its own: a release for an id a replacement owner has since reused is rejected by the
1589
+ * transport, which stamps every release with the owner session captured when the admission was minted
1590
+ * and drops its cached sessions when the coordinating thread changes. An owner ignores anything not
1591
+ * stamped with its own session, so a stale release cannot address a live admission that reuses the id.
1592
+ */
1593
+ #releaseOnOwnerSafely(key: any, ownerAdmissionId: number): void {
1594
+ // The common relayed unlock is a synchronous post to the owner thread, so it must not cost a
1595
+ // promise chain and two microtasks per unlock: call it directly and only attach a rejection
1596
+ // handler when the transport actually returned a promise.
1597
+ try {
1598
+ const outcome = this.transport.releaseOnOwner?.(this.database, this.table, key, ownerAdmissionId);
1599
+ if (isPromiseLike(outcome))
1600
+ outcome.catch((error) => warnOnce('failed to forward a relayed record lock release to the owner', error));
1601
+ } catch (error) {
1602
+ warnOnce('failed to forward a relayed record lock release to the owner', error);
1603
+ }
1604
+ }
1605
+
1606
+ /** Resolve every ack waiting on a remote admission's fence, then forget them. */
1607
+ #settleFenceWaiters(remote: RemoteAdmission): void {
1608
+ if (remote.fenceWaiters.length === 0) return;
1609
+ const waiters = remote.fenceWaiters;
1610
+ remote.fenceWaiters = [];
1611
+ for (const resolve of waiters) resolve();
1612
+ }
1613
+
1614
+ /**
1615
+ * Drop remote admissions whose handle lease elapsed a grace ago, at most `MAX_EXPIRIES_PER_TICK` per
1616
+ * tick like the delegation and grant sweeps — a burst of tens of thousands of same-lease admissions
1617
+ * expiring together drains over several ticks rather than deleting them all (and firing a release
1618
+ * message apiece) in one 100 ms turn. The scan skips still-live entries so different lease lengths are
1619
+ * handled. `REMOTE_PRUNE_GRACE_MS` past the lease is what lets the handle's OWN lease timer fire first
1620
+ * and forward its release (`release`) rather than racing this sweep.
1621
+ *
1622
+ * Entries EXAMINED are bounded too, not just deletions: a thread holding many live relayed admissions
1623
+ * would otherwise walk every one of them on every 100 ms tick, purely to find nothing. What is left
1624
+ * unexamined is reached on a later tick, which is safe because this sweep only reclaims memory — an
1625
+ * admission's handle is fenced by its own lease, never by this loop running promptly.
1626
+ *
1627
+ * The scan restarts at the map head each tick rather than carrying a cursor, so above the examine cap
1628
+ * a long-lived prefix would delay collecting expired entries behind it. Accepted deliberately: the cap
1629
+ * is far above any realistic count of concurrent off-owner locks on one thread, and a cursor would
1630
+ * have to walk the prefix anyway to find its resume point, spending the work it meant to save. It
1631
+ * costs memory held longer in a case that should not arise, never correctness.
1632
+ */
1633
+ #pruneRemoteAdmissions(now: number): void {
1634
+ const remotes = this.#remoteAdmissions;
1635
+ if (!remotes) return;
1636
+ const horizon = now - REMOTE_PRUNE_GRACE_MS;
1637
+ let budget = MAX_EXPIRIES_PER_TICK;
1638
+ let examined = MAX_EXAMINED_PER_TICK;
1639
+ for (const [localId, remote] of remotes) {
1640
+ if (budget <= 0 || examined <= 0) break;
1641
+ examined--;
1642
+ if (remote.expiresMono > horizon) continue;
1643
+ budget--;
1644
+ this.#dropRemoteAdmission(localId, remote, true);
1645
+ }
1646
+ if (remotes.size === 0) {
1647
+ this.#remoteAdmissions = undefined;
1648
+ this.#remoteByOwnerId = undefined;
1649
+ }
1650
+ }
1651
+
1652
+ /**
1653
+ * Drop a remote admission entry. `forwardRelease` tells the owner to give the delegation up now —
1654
+ * set when pruning at lease, since an event-loop stall can let this sweep run before the handle's
1655
+ * own lease timer forwards the release, and the owner would otherwise hold `holding` until its own
1656
+ * delegation lease. The forward is idempotent on the owner, so a race with the handle's own release
1657
+ * is harmless. Not set for a fence-drop: the owner is already surrendering that delegation.
1658
+ */
1659
+ #dropRemoteAdmission(localId: number, remote: RemoteAdmission, forwardRelease = false): void {
1660
+ // Optional: an asynchronous fence can settle after `fenceAllRemoteAdmissions`, `close` or a
1661
+ // handoff has already cleared the map, and a TypeError here would reject an ack whose handle is
1662
+ // in fact fenced, making the owner wait out the lease for nothing.
1663
+ this.#remoteAdmissions?.delete(localId);
1664
+ this.#remoteByOwnerId?.delete(remote.ownerAdmissionId);
1665
+ // The handle's own lease has elapsed, so it fences itself; a still-pending ack may resolve.
1666
+ this.#settleFenceWaiters(remote);
1667
+ if (forwardRelease) this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
1668
+ }
1669
+
1670
+ /**
1671
+ * Resolve and drop a latched revoke whose admission never installed — a revoke for an admission this
1672
+ * thread had already dropped (fenced or pruned), so the handle is gone and the ack is vacuously
1673
+ * satisfied. Bounded by a lease: past that, no acquire can still be in flight for the id.
1674
+ */
1675
+ #prunePendingRemoteRevokes(now: number): void {
1676
+ const pending = this.#pendingRemoteRevokes;
1677
+ if (!pending) return;
1678
+ // Bounded like the admission sweep above, and for the same reason: what this tick does not reach,
1679
+ // a later one does. Each waiting ack is independently bounded by the owner's own lease wait.
1680
+ let examined = MAX_EXAMINED_PER_TICK;
1681
+ for (const [ownerId, entry] of pending) {
1682
+ if (examined <= 0) break;
1683
+ examined--;
1684
+ if (now - entry.at < MAX_LOCK_LEASE_MS) continue;
1685
+ pending.delete(ownerId);
1686
+ for (const resolve of entry.resolvers) resolve();
1687
+ }
1688
+ if (pending.size === 0) this.#pendingRemoteRevokes = undefined;
1689
+ }
1690
+
1691
+ /**
1692
+ * End this node's admission for a key. The delegation is deliberately KEPT: that is the
1693
+ * amortization, and the next `lock()` on this node costs nothing. Returns the durable release
1694
+ * write only when the delegation is actually being given up.
1695
+ */
1696
+ release(key: any, admissionId: number): Promise<void> | void {
1697
+ // A remote admission (harper-pro#852) — the common path once threads.count > 1 — tells the owner
1698
+ // this caller unlocked (its own id, not this thread's local one) but keeps the entry, revoker
1699
+ // included, until the handle's lease runs out: §6 revokes CAPABILITY, not admission, so a write
1700
+ // staged before `unlock()` stays fenceable. Checked before `#keyIdOf` so a relayed unlock does not
1701
+ // pay that key encoding for nothing.
1702
+ const remote = this.#remoteAdmissions?.get(admissionId);
1703
+ if (remote) {
1704
+ this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
1705
+ return undefined;
1706
+ }
1707
+ const keyId = this.#keyIdOf(key);
1708
+ // Addressed by admission, not by key: after a renewal or a replacement the delegation at this
1709
+ // key may not be the one that admitted this handle, and releasing by key alone would either
1710
+ // surrender a successor while its own callers were still inside, or leave this admission on a
1711
+ // delegation nobody can ever drain.
1712
+ const delegation = this.#admissions.get(admissionId);
1713
+ if (!delegation) return undefined;
1714
+ const admission = delegation.admissions.get(admissionId);
1715
+ if (!admission?.holding) return undefined;
1716
+ // The entry stays. Unlocking ends this caller's claim on the DRAIN; the write it staged can
1717
+ // still commit, so the revoker has to remain reachable until the handle's own lease runs out.
1718
+ admission.holding = false;
1719
+ delegation.holding--;
1720
+ if (delegation.holding > 0) return undefined;
1721
+ if (delegation.drained) {
1722
+ // Waking the recall is enough: it surrenders once, on its own path. Surrendering here too
1723
+ // would write a second release entry for the same handoff.
1724
+ const waiters = delegation.drained;
1725
+ delegation.drained = undefined;
1726
+ for (const resolve of waiters) resolve();
1727
+ return undefined;
1728
+ }
1729
+ // Recalled with nobody waiting on the drain — the recall already returned, so this is the path
1730
+ // that gives the delegation up. An un-recalled delegation is deliberately retained.
1731
+ if (delegation.recalled && this.#delegations.get(keyId) === delegation) return this.#surrender(keyId, delegation);
1732
+ return undefined;
1733
+ }
1734
+
1735
+ /**
1736
+ * Apply a control entry from a peer (or a replayed one). Idempotent by `(requester, tsR)`: a
1737
+ * release that arrives twice, or is replayed from the log long after its producer is gone, must be
1738
+ * harmless.
1739
+ *
1740
+ * `author` is the node the entry was actually written by, taken from the audit header rather than
1741
+ * the payload — without it a peer could write a release naming any other node and clear a
1742
+ * delegation it does not hold.
1743
+ */
1744
+ applyEntry(entry: LockControlEntry, author: string, position?: number): void {
1745
+ // The only boundary peer input crosses into this state machine. A throw here would reach the
1746
+ // replicated apply loop and drop the whole enclosing transaction, so one malformed entry could
1747
+ // stall replication for the database.
1748
+ try {
1749
+ this.#applyEntry(entry, author, position);
1750
+ } catch (error) {
1751
+ warnOnce('failed to apply a record lock control entry', error);
1752
+ }
1753
+ }
1754
+
1755
+ #applyEntry(entry: LockControlEntry, author: string, position?: number): void {
1756
+ if (this.#closed) return;
1757
+ if (entry.type !== 'lockRelease' || !isNodeName(author)) return;
1758
+ if (entry.requester !== author) return;
1759
+ if (!this.transport.ownsCoordination()) {
1760
+ this.#droppedOffOwner++;
1761
+ const now = this.#monotonic();
1762
+ if (now - this.#lastOffOwnerWarn > WARN_INTERVAL_MS) {
1763
+ this.#lastOffOwnerWarn = now;
1764
+ harperLogger.warn?.('record lock control entries are reaching a non-coordinating thread', {
1765
+ database: this.database,
1766
+ table: this.table,
1767
+ dropped: this.#droppedOffOwner,
1768
+ });
1769
+ }
1770
+ return;
1771
+ }
1772
+ if (!isFencingToken(entry.token)) return;
1773
+ const keyId = this.#keyIdOf(entry.key);
1774
+ const grant = this.#grants.get(keyId);
1775
+ // Only the delegate named in the live grant can clear it, and only for the exact token it was
1776
+ // issued — generation and incarnation included, since counters restart. A delayed release from a
1777
+ // previous delegation must not clear its successor's.
1778
+ if (!grant || grant.delegate !== author || compareTokens(grant.token, entry.token) !== 0) return;
1779
+ if (entry.dependencies !== null) {
1780
+ const homeMap = this.transport.homeMap(this.database);
1781
+ const matchingHomes = homeMap?.generation === entry.token[0] ? homeMap.homes : undefined;
1782
+ const inherited = matchingHomes ? normalizeDependencies(entry.dependencies, matchingHomes) : undefined;
1783
+ if (
1784
+ inherited &&
1785
+ matchingHomes.includes(author) &&
1786
+ typeof position === 'number' &&
1787
+ Number.isFinite(position) &&
1788
+ position >= 0
1789
+ ) {
1790
+ const merged = new Map<string, number>(this.#dependencySets.get(keyId));
1791
+ for (const [origin, dependencyPosition] of inherited)
1792
+ merged.set(origin, Math.max(merged.get(origin) ?? -Infinity, dependencyPosition));
1793
+ merged.set(author, Math.max(merged.get(author) ?? -Infinity, position));
1794
+ if (merged.size <= MAX_LOCK_DEPENDENCIES)
1795
+ this.#rememberDependencies(
1796
+ keyId,
1797
+ [...merged].sort(([a], [b]) => a.localeCompare(b))
1798
+ );
1799
+ else this.#dependencySets.delete(keyId);
1800
+ } else {
1801
+ this.#dependencySets.delete(keyId);
1802
+ }
1803
+ } else if (grant.renewed) this.#dependencySets.delete(keyId);
1804
+ this.#clearGrant(keyId, grant);
1805
+ }
1806
+
1807
+ /**
1808
+ * Inbound delegation request from a peer, for a key this node homes. The home is the single
1809
+ * arbiter, so this is the whole of the exclusion argument: one live grant per key, and a successor
1810
+ * only after the predecessor is recalled-and-drained or provably expired here.
1811
+ */
1812
+ async onDelegationRequest(request: DelegationRequest): Promise<DelegationReply> {
1813
+ if (this.#closed || !this.transport.ownsCoordination()) return { granted: false, reason: 'not-home' };
1814
+ if (!isNodeName(request.requester) || !isEncodableKey(request.key)) return { granted: false, reason: 'not-home' };
1815
+ if (!isDuration(request.leaseMs, MIN_LOCK_LEASE_MS, MAX_LOCK_LEASE_MS))
1816
+ return { granted: false, reason: 'not-home' };
1817
+ const homeMap = this.transport.homeMap(this.database);
1818
+ if (!homeMap || !this.#generationIsCurrent(homeMap.generation)) return { granted: false, reason: 'generation' };
1819
+ if (homeMap.generation !== request.generation)
1820
+ return { granted: false, reason: 'generation', generation: homeMap.generation };
1821
+ // Membership before state: a node the current map does not name has no claim on a key, and an
1822
+ // authenticated replication identity outlives membership. Without this a decommissioned node
1823
+ // takes delegations against live homes and recalls the legitimate delegate to get them. This is
1824
+ // why `homes` has to name every node that takes a cluster lock, not only the arbiters — see
1825
+ // `LockHomeMap.homes`.
1826
+ if (!homeMap.homes.includes(request.requester)) return { granted: false, reason: 'unknown-node' };
1827
+ // Both sides must agree we are the home, or two arbiters could issue for one key.
1828
+ const keyId = this.#keyIdOf(request.key);
1829
+ if (homeFor(this.#ringKey(keyId), homeMap.homes) !== this.nodeId) return { granted: false, reason: 'not-home' };
1830
+ return this.#grant(keyId, request.key, homeMap, request.leaseMs, request.requester);
1831
+ }
1832
+
1833
+ /**
1834
+ * Inbound recall from a key's home. Revokes capability rather than closing the door: no new
1835
+ * admission may start, live ones are drained, and the release is written only once the delegation
1836
+ * can no longer admit or commit. A recall for a token we no longer hold is a no-op, not an error.
1837
+ */
1838
+ async onDelegationRecall(recall: DelegationRecall): Promise<void> {
1839
+ // A close that was not a handoff expired and revoked every delegation first, so nothing here can
1840
+ // admit on any token and resolving is honest. A handoff does not reach this: the table's getter
1841
+ // answers the successor, which adopted them.
1842
+ if (this.#closed) return;
1843
+ // Ownership-gated like `onDelegationRequest`, and for the stronger reason: `acquire` refuses off
1844
+ // the owner thread, so a delegation only ever lives on the coordinating one. A recall routed to
1845
+ // any other thread finds no delegation and would resolve — which the home reads as a drained
1846
+ // delegate — while the real delegate keeps admitting. Resolving has to mean nothing on THIS NODE
1847
+ // can admit on that token, and only the owner can say so.
1848
+ if (!this.transport.ownsCoordination())
1849
+ throw new Error('Cluster record lock coordination is not owned by this worker thread');
1850
+ const keyId = this.#keyIdOf(recall.key);
1851
+ const pending = this.#pendingDelegations.get(keyId);
1852
+ if (pending && compareTokens(pending.token, recall.token) === 0) {
1853
+ pending.markRecalled();
1854
+ return;
1855
+ }
1856
+ const delegation = this.#delegations.get(keyId);
1857
+ if (!delegation || compareTokens(delegation.token, recall.token) !== 0) {
1858
+ // A recall can beat the grant reply. Remember its token so that reply cannot install authority
1859
+ // the home already believes drained.
1860
+ const request = this.#pendingRequests.get(keyId);
1861
+ if (request) request.recalledToken = recall.token;
1862
+ return;
1863
+ }
1864
+ delegation.recalled = true;
1865
+ if (delegation.holding > 0) {
1866
+ await new Promise<void>((resolve) => {
1867
+ // A holder that never releases must not hold the recall open past its own lease: the
1868
+ // delegation expires here on our clock, and the home outwaits that by skew anyway. The
1869
+ // timer is dropped by whichever side wins, so a delegation that drains promptly does not
1870
+ // retain one for the rest of its lease.
1871
+ const remaining = Math.max(0, delegation.expiresMono - this.#monotonic());
1872
+ const timer = setTimeout(resolve, remaining);
1873
+ timer.unref?.();
1874
+ (delegation.drained ||= []).push(() => {
1875
+ clearTimeout(timer);
1876
+ resolve();
1877
+ });
1878
+ });
1879
+ }
1880
+ // The swap may have landed during the drain. The delegation OBJECT moved to the successor, so
1881
+ // surrendering here would revoke through an index this coordinator no longer owns and leave the
1882
+ // successor still holding a delegation whose release has already been written.
1883
+ await this.#authority().#surrender(keyId, delegation);
1884
+ }
1885
+
1886
+ /** Expire delegations and grants whose deadlines have passed. Bounded work per call. */
1887
+ tick(): void {
1888
+ const now = this.#monotonic();
1889
+ // A second, weaker signal than the incarnation: it catches a thread that stopped coordinating
1890
+ // without anything else starting, which advances no incarnation. Rate-limited well below the
1891
+ // tick because it cannot matter faster than a delegation lease, and it costs nothing at all on a
1892
+ // coordinator holding nothing — that one has already stopped ticking.
1893
+ if (now - this.#lastOwnershipPollMono >= OWNERSHIP_POLL_MS) {
1894
+ this.#lastOwnershipPollMono = now;
1895
+ this.#observeOwnership();
1896
+ }
1897
+ this.#pruneRemoteAdmissions(now);
1898
+ this.#prunePendingRemoteRevokes(now);
1899
+ // The budget counts EXPIRIES, not entries examined. Spending it on live entries would let a
1900
+ // table with more than a budget's worth of continuously renewed grants starve every expired
1901
+ // one behind them, and an uncollected expired grant answers `contended` to every other node.
1902
+ let budget = MAX_EXPIRIES_PER_TICK;
1903
+ for (const [keyId, delegation] of this.#delegations) {
1904
+ if (budget <= 0) break;
1905
+ if (delegation.expiresMono > now) continue;
1906
+ budget--;
1907
+ // A delegate may drop early; the handles it admitted are fenced by their own lease, which
1908
+ // never outlives the delegation that admitted them.
1909
+ this.#delegations.delete(keyId);
1910
+ this.#revokeAll(delegation);
1911
+ const waiters = delegation.drained;
1912
+ if (waiters) {
1913
+ delegation.drained = undefined;
1914
+ for (const resolve of waiters) resolve();
1915
+ }
1916
+ }
1917
+ budget = MAX_EXPIRIES_PER_TICK;
1918
+ for (const [keyId, grant] of this.#grants) {
1919
+ if (budget <= 0) break;
1920
+ // A home may NEVER forget a grant before its expiry — that asymmetry is the safety rule.
1921
+ if (grant.expiresMono > now) continue;
1922
+ budget--;
1923
+ this.#expireGrant(keyId, grant);
1924
+ }
1925
+ if (
1926
+ this.#delegations.size === 0 &&
1927
+ this.#grants.size === 0 &&
1928
+ !this.#remoteAdmissions &&
1929
+ !this.#pendingRemoteRevokes
1930
+ )
1931
+ tickingCoordinators.delete(this);
1932
+ }
1933
+
1934
+ /**
1935
+ * One table's half of `quiesceDelegations`. Never throws for a single grant: a transition needs to
1936
+ * know exactly what is still live, and one unreachable delegate must not hide the rest.
1937
+ */
1938
+ async quiesce(result: QuiesceResult, remaining: () => number): Promise<void> {
1939
+ if (this.#closed) return;
1940
+ // Delegate side first: this is what admits, and surrendering is purely local — it cannot be
1941
+ // refused by an unreachable peer, so it succeeds even when the recalls below do not.
1942
+ for (const delegation of [...this.#delegations.values()]) {
1943
+ try {
1944
+ await withDeadline(this.onDelegationRecall({ key: delegation.key, token: delegation.token }), remaining());
1945
+ result.surrendered++;
1946
+ } catch (error) {
1947
+ result.outstanding.push({
1948
+ table: this.table,
1949
+ key: delegation.key,
1950
+ reason: `this node still holds a delegation it could not drain: ${(error as Error)?.message ?? error}`,
1951
+ });
1952
+ }
1953
+ }
1954
+ // Home side: tell delegates elsewhere to stop. `#beginRecall` owns the retry and confirmation
1955
+ // bookkeeping; this only drives it and reports what it did not confirm.
1956
+ for (const [keyId, grant] of [...this.#grants]) {
1957
+ if (grant.recallConfirmed) {
1958
+ result.recalled++;
1959
+ continue;
1960
+ }
1961
+ try {
1962
+ this.#beginRecall(keyId, grant);
1963
+ if (grant.recalling) await withDeadline(grant.recalling, remaining());
1964
+ if (grant.recallConfirmed) result.recalled++;
1965
+ else
1966
+ result.outstanding.push({
1967
+ table: this.table,
1968
+ key: grant.key,
1969
+ delegate: grant.delegate,
1970
+ reason: 'the delegate did not confirm it stopped admitting',
1971
+ });
1972
+ } catch (error) {
1973
+ result.outstanding.push({
1974
+ table: this.table,
1975
+ key: grant.key,
1976
+ delegate: grant.delegate,
1977
+ reason: `recall failed: ${(error as Error)?.message ?? error}`,
1978
+ });
1979
+ }
1980
+ }
1981
+ }
1982
+
1983
+ /**
1984
+ * Stop this coordinator and invalidate what it issued. Expiring every delegation is the part that
1985
+ * matters: `close()` runs when a transport is replaced (a component reload is enough), and a
1986
+ * successor coordinator must not be able to grant a key whose predecessor handles are still live.
1987
+ */
1988
+ close(): void {
1989
+ this.#closed = true;
1990
+ liveCoordinators.delete(this);
1991
+ // State that was handed to a successor is that coordinator's now; expiring it here would
1992
+ // invalidate delegations the successor is correctly still honouring.
1993
+ if (this.#handedOff) {
1994
+ tickingCoordinators.delete(this);
1995
+ return;
1996
+ }
1997
+ for (const delegation of this.#delegations.values()) {
1998
+ delegation.recalled = true;
1999
+ delegation.expiresMono = -Infinity;
2000
+ this.#revokeAll(delegation);
2001
+ const waiters = delegation.drained;
2002
+ if (waiters) {
2003
+ delegation.drained = undefined;
2004
+ for (const resolve of waiters) resolve();
2005
+ }
2006
+ }
2007
+ for (const pending of this.#pendingDelegations.values()) pending.markRecalled();
2008
+ // Remote admissions (harper-pro#852) that were NOT handed to a successor: this thread is going
2009
+ // away, so fence their handles (fail closed) and tell the owner it may give the delegation up now
2010
+ // rather than hold it for a full lease waiting on a worker that has gone.
2011
+ if (this.#remoteAdmissions) {
2012
+ for (const remote of this.#remoteAdmissions.values()) {
2013
+ fireRevokeAndForget(remote.revoke);
2014
+ this.#settleFenceWaiters(remote);
2015
+ this.#releaseOnOwnerSafely(remote.key, remote.ownerAdmissionId);
2016
+ }
2017
+ this.#remoteAdmissions = undefined;
2018
+ this.#remoteByOwnerId = undefined;
2019
+ }
2020
+ // Any ack still waiting on an entry that never installed cannot be honored here; resolve it so the
2021
+ // owner's revoke wait does not hang on a thread that is gone (its own lease bounds it regardless).
2022
+ if (this.#pendingRemoteRevokes) {
2023
+ for (const entry of this.#pendingRemoteRevokes.values()) for (const resolve of entry.resolvers) resolve();
2024
+ this.#pendingRemoteRevokes = undefined;
2025
+ }
2026
+ // The delegations THIS node issued as a home outlive it: no peer sees a local close, so each one
2027
+ // stands until its own deadline. Leave the latest of those deadlines, and the counter, for
2028
+ // whatever coordinator takes this table next.
2029
+ let grantableAfterMono = -Infinity;
2030
+ for (const grant of this.#grants.values()) {
2031
+ // A grant to THIS node is settled by the same close: the loop above revoked the delegation it
2032
+ // authorized. Only what another node holds outlives this coordinator unseen.
2033
+ if (grant.delegate === this.nodeId) continue;
2034
+ if (grant.expiresMono > grantableAfterMono) grantableAfterMono = grant.expiresMono;
2035
+ }
2036
+ const retirementKey = this.#retirementKey();
2037
+ const retired = retiredCoordinators.get(retirementKey);
2038
+ retiredCoordinators.set(retirementKey, {
2039
+ grantableAfterMono: Math.max(retired?.grantableAfterMono ?? -Infinity, grantableAfterMono),
2040
+ counter: Math.max(retired?.counter ?? 0, this.#counter),
2041
+ });
2042
+ this.#delegations.clear();
2043
+ this.#pendingDelegations.clear();
2044
+ this.#pendingRequests.clear();
2045
+ this.#grants.clear();
2046
+ this.#grantsByRequester.clear();
2047
+ this.#dependencySets.clear();
2048
+ this.#everDelegated.clear();
2049
+ tickingCoordinators.delete(this);
2050
+ }
2051
+
2052
+ /**
2053
+ * Ownership as this thread can actually observe it. Only a LOSS is recorded: regaining it is the
2054
+ * grant path's business, because that is where the incarnation is available to say whether anything
2055
+ * else coordinated in between.
2056
+ */
2057
+ #observeOwnership(): void {
2058
+ let owns: boolean;
2059
+ try {
2060
+ owns = this.transport.ownsCoordination();
2061
+ } catch {
2062
+ // A transport that cannot answer is not proof this thread kept coordinating, and the whole
2063
+ // point of the horizon is what happened while it did not.
2064
+ owns = false;
2065
+ }
2066
+ if (!owns) this.#ownedSinceMono = undefined;
2067
+ }
2068
+
2069
+ /**
2070
+ * The horizon owning coordination imposes, anchored at the instant this coordinator STARTED
2071
+ * coordinating under the incarnation it is granting under — see `#ownedSinceMono`.
2072
+ *
2073
+ * `homeIncarnation` is the signal, not `ownsCoordination()`. Sampling a boolean proves the answer
2074
+ * at the instant it is read and never that ownership was unbroken between two reads, so ownership
2075
+ * alternating faster than the sample interval could alias away entirely. §5.1 makes the incarnation
2076
+ * advance once per COORDINATION incarnation — that is what keeps the fencing token orderable — so a
2077
+ * value this coordinator has not granted under is the transport stating that something else
2078
+ * coordinated for this node, whatever the boolean said in between.
2079
+ */
2080
+ #ownershipHorizon(now: number, homeIncarnation: number): number {
2081
+ if (this.#coordinatingIncarnation !== homeIncarnation || this.#ownedSinceMono === undefined) {
2082
+ this.#loseFreshnessHistory();
2083
+ // Ownership STARTED here, which is the one thing the waiver cannot cover.
2084
+ // `grantableAfterMono` attests that no previous INCARNATION OF THIS PROCESS had issued
2085
+ // delegations — a cold start, a fresh database, a test — and says nothing about the sibling
2086
+ // thread that was coordinating until this instant. The constructor records both directly, so
2087
+ // a coordinator that has coordinated since it was built never reaches this and keeps the
2088
+ // waiver; every other way of arriving at ownership lands here and loses it.
2089
+ this.#coordinatingIncarnation = homeIncarnation;
2090
+ this.#quarantineWaived = false;
2091
+ this.#ownedSinceMono = now;
2092
+ }
2093
+ if (this.#quarantineWaived) return -Infinity;
2094
+ return this.#ownedSinceMono + DELEGATION_LEASE_MS + this.#skewMs;
2095
+ }
2096
+
2097
+ /** Fail closed on a generation older than one already ACTED on here (see `highestGeneration`). */
2098
+ #generationIsCurrent(generation: number): boolean {
2099
+ const highest = highestGeneration.get(this.database);
2100
+ return highest === undefined || generation >= highest;
2101
+ }
2102
+
2103
+ /**
2104
+ * Raise the floor, at the moment authority is actually taken under this generation — a token
2105
+ * minted or a delegation installed — and never merely on reading a map.
2106
+ *
2107
+ * Observing was the obvious place and it is the wrong one: a single `homeMap()` that returns a
2108
+ * too-large generation once, from a partial publish or a transport glitch, would pin the floor
2109
+ * above anything the operator ever publishes and fail every later lock on this database until the
2110
+ * thread restarts. Nothing was minted under that reading, so nothing needs protecting from it. The
2111
+ * invariant only ever needed to be "never mint below a generation already minted".
2112
+ */
2113
+ #recordGenerationActedOn(generation: number): void {
2114
+ const highest = highestGeneration.get(this.database);
2115
+ if (highest === undefined || generation > highest) highestGeneration.set(this.database, generation);
2116
+ }
2117
+
2118
+ #prepareFreshnessGeneration(generation: number): void {
2119
+ if (this.#freshnessGeneration === undefined) {
2120
+ this.#freshnessGeneration = generation;
2121
+ return;
2122
+ }
2123
+ if (this.#freshnessGeneration === generation) return;
2124
+ this.#freshnessGeneration = generation;
2125
+ this.#loseFreshnessHistory();
2126
+ }
2127
+
2128
+ #loseFreshnessHistory(): void {
2129
+ this.#dependencySets.clear();
2130
+ this.#everDelegated.clear();
2131
+ this.#trustVirginKeys = false;
2132
+ }
2133
+
2134
+ #freshnessFor(keyId: unknown): LockDependencySet | null {
2135
+ const retained = this.#dependencySets.get(keyId);
2136
+ if (retained) {
2137
+ this.#dependencySets.delete(keyId);
2138
+ this.#dependencySets.set(keyId, retained);
2139
+ return retained;
2140
+ }
2141
+ return this.#trustVirginKeys && !this.#everDelegated.has(keyId) ? [] : null;
2142
+ }
2143
+
2144
+ #rememberDependencies(keyId: unknown, dependencies: LockDependencySet): void {
2145
+ this.#dependencySets.delete(keyId);
2146
+ this.#dependencySets.set(keyId, dependencies);
2147
+ while (this.#dependencySets.size > MAX_DEPENDENCY_SETS_PER_TABLE)
2148
+ this.#dependencySets.delete(this.#dependencySets.keys().next().value);
2149
+ }
2150
+
2151
+ #retirementKey(): string {
2152
+ return `${this.database}\u0000${this.table}`;
2153
+ }
2154
+
2155
+ #ringKey(keyId: unknown): string {
2156
+ return ringKeyFor(this.database, this.table, keyId);
2157
+ }
2158
+
2159
+ /** The coordinator holding this node's authority now: this one, or the end of the handoff chain. */
2160
+ #authority(): LockCoordinator {
2161
+ let coordinator: LockCoordinator = this;
2162
+ while (coordinator.#successor) coordinator = coordinator.#successor;
2163
+ return coordinator;
2164
+ }
2165
+
2166
+ #liveDelegation(keyId: unknown, leaseMs: number, generation: number): Delegation | undefined {
2167
+ const delegation = this.#delegations.get(keyId);
2168
+ if (!delegation || delegation.recalled) return undefined;
2169
+ // A delegation is authority within ONE generation. After a generation change the key may have
2170
+ // been re-homed, and the new home knows nothing of this token — so keeping it would let this
2171
+ // node admit alongside whoever the new home grants.
2172
+ if (delegation.token[0] !== generation) {
2173
+ // Authority is gone, not merely stale: the key may have been re-homed to a node that knows
2174
+ // nothing of this token. Forgetting the delegation without revoking would leave its handles
2175
+ // able to commit alongside whatever the new home grants.
2176
+ this.#delegations.delete(keyId);
2177
+ this.#revokeAll(delegation);
2178
+ return undefined;
2179
+ }
2180
+ // The admission may not outlive the delegation that admitted it, so a delegation without room
2181
+ // for the whole lease is renewed rather than stretched.
2182
+ if (delegation.expiresMono - this.#monotonic() < leaseMs) return undefined;
2183
+ return delegation;
2184
+ }
2185
+
2186
+ #admit(delegation: Delegation, leaseMs: number): LockRound {
2187
+ const mintedMono = this.#monotonic();
2188
+ // Admissions that unlocked are kept only until their handle's own lease fences it. Collecting
2189
+ // them here rather than on a timer keeps the work on the path that creates it, and a delegation
2190
+ // can accumulate at most one entry per overlapping lock in its own lease.
2191
+ this.#pruneAdmissions(delegation, mintedMono);
2192
+ const admissionId = this.#nextAdmissionId++;
2193
+ // The entry exists from the instant of admission, so a recall between admit and register still
2194
+ // sees it.
2195
+ delegation.admissions.set(admissionId, { revoke: noRevoke, expiresMono: mintedMono + leaseMs, holding: true });
2196
+ delegation.holding++;
2197
+ this.#admissions.set(admissionId, delegation);
2198
+ return { tsR: this.#nextTimestamp(), mintedMono, admissionId };
2199
+ }
2200
+
2201
+ /**
2202
+ * Drop admissions whose handle's lease has run out; they can no longer commit anything.
2203
+ *
2204
+ * Insertion order is monotonic order, not expiry order: under one lease length the leading run IS
2205
+ * the expired set, but a longer-lease admission at the head hides every shorter one behind it for
2206
+ * its own remaining lease. Scanning the whole map on every admission instead would be quadratic on
2207
+ * the hot key this design exists to make cheap, so it is swept only once it has outgrown the live
2208
+ * set the previous sweep measured — amortized O(1) per admission, and the map stays within twice
2209
+ * the live set rather than growing at the lock rate.
2210
+ *
2211
+ * Never has a drain to wake: `#admit` reaches neither call site for a recalled delegation.
2212
+ */
2213
+ #pruneAdmissions(delegation: Delegation, now: number): void {
2214
+ for (const [admissionId, admission] of delegation.admissions) {
2215
+ if (admission.expiresMono > now) break;
2216
+ this.#dropAdmission(delegation, admissionId, admission);
2217
+ }
2218
+ if (delegation.admissions.size < delegation.sweepAtSize) return;
2219
+ for (const [admissionId, admission] of delegation.admissions)
2220
+ if (admission.expiresMono <= now) this.#dropAdmission(delegation, admissionId, admission);
2221
+ delegation.sweepAtSize = delegation.admissions.size * 2 + ADMISSION_SWEEP_FLOOR;
2222
+ }
2223
+
2224
+ #dropAdmission(delegation: Delegation, admissionId: number, admission: Admission): void {
2225
+ delegation.admissions.delete(admissionId);
2226
+ this.#admissions.delete(admissionId);
2227
+ if (admission.holding) delegation.holding--;
2228
+ }
2229
+
2230
+ /**
2231
+ * Register how to revoke the handle an admission produced. `Table.lock()` calls this once the
2232
+ * handle has joined the round, so a recall can fence a write that was staged and then unlocked.
2233
+ */
2234
+ registerAdmission(admissionId: number, revoke: () => void | Promise<void>): void {
2235
+ const admission = this.#admissions.get(admissionId)?.admissions.get(admissionId);
2236
+ if (admission) {
2237
+ admission.revoke = revoke;
2238
+ return;
2239
+ }
2240
+ // A remote admission (harper-pro#852): the delegation lives on the owner, the handle here. A revoke
2241
+ // that already latched (it beat this registration) fences the handle now and releases the ack that
2242
+ // was waiting on the fence; otherwise store the revoker for a later recall.
2243
+ const remote = this.#remoteAdmissions?.get(admissionId);
2244
+ if (remote) {
2245
+ remote.revoke = revoke;
2246
+ if (remote.revoked) {
2247
+ // A revoke latched before this registration: fence now, then drop the entry and release the
2248
+ // ack that was waiting on the fence. `#dropRemoteAdmission` is what resolves that ack, so an
2249
+ // async revoker must settle FIRST — resolving it early would tell the owner this handle is
2250
+ // fenced while it still is not, and the owner would write the release over a live writer. A fence
2251
+ // that FAILS settles nothing: the entry is left for the lease sweep, which resolves the ack once
2252
+ // the handle is provably dead. The failure is warned about inside and has nobody here to go to,
2253
+ // so it is absorbed rather than left to Node's unhandled-rejection policy.
2254
+ const fenced = this.#fenceRemoteAdmission(admissionId, remote);
2255
+ if (isPromiseLike(fenced)) fenced.catch(() => {});
2256
+ }
2257
+ return;
2258
+ }
2259
+ // The admission was already revoked or collected between admit and register — the handle has no
2260
+ // authority to keep, so revoke it now (safely: a relay revoker collected in this window is async
2261
+ // and could reject on a dead port) rather than leaving it unfenced.
2262
+ fireRevokeAndForget(revoke);
2263
+ }
2264
+
2265
+ /**
2266
+ * Install a granted delegation, with its deadline anchored at the moment the request was SENT
2267
+ * rather than at the moment the reply arrived. The home started its own clock when it granted, so
2268
+ * anchoring on arrival would hand a delayed reply more time than the home is holding the key for —
2269
+ * and a reply delayed past the whole delegation must be discarded, not installed (§5.2).
2270
+ */
2271
+ #installDelegation(
2272
+ keyId: unknown,
2273
+ key: any,
2274
+ token: FencingToken,
2275
+ leaseMs: number,
2276
+ requestedAtMono: number,
2277
+ dependencies: LockDependencySet
2278
+ ): Delegation | undefined {
2279
+ const expiresMono = requestedAtMono + leaseMs;
2280
+ if (expiresMono <= this.#monotonic()) return undefined;
2281
+ // The delegate side of taking authority under a generation: installing this makes it admit under
2282
+ // `token[0]`, so that is the floor a later map may not go below.
2283
+ this.#recordGenerationActedOn(token[0]);
2284
+ const existing = this.#delegations.get(keyId);
2285
+ // A reply that lost a race with a newer delegation for the same key must not move it backwards.
2286
+ if (existing && compareTokens(existing.token, token) >= 0) return existing;
2287
+ if (existing && !existing.recalled) {
2288
+ // A RENEWAL of authority this node never lost. Advance the token and the deadline in place so
2289
+ // the admissions it is still answerable for ride along; installing a fresh object here
2290
+ // orphaned them, and the next recall then surrendered while a live handle could still commit.
2291
+ existing.token = token;
2292
+ existing.expiresMono = expiresMono;
2293
+ this.#startTicking();
2294
+ return existing;
2295
+ }
2296
+ // Replacing a recalled delegation: its handles were revoked at surrender, so nothing carries.
2297
+ if (existing) this.#revokeAll(existing);
2298
+ const delegation: Delegation = {
2299
+ key,
2300
+ token,
2301
+ dependencies,
2302
+ expiresMono,
2303
+ recalled: false,
2304
+ admissions: new Map(),
2305
+ holding: 0,
2306
+ sweepAtSize: ADMISSION_SWEEP_FLOOR,
2307
+ };
2308
+ this.#delegations.set(keyId, delegation);
2309
+ this.#startTicking();
2310
+ return delegation;
2311
+ }
2312
+
2313
+ async #establishFreshness(
2314
+ key: any,
2315
+ requirement: LockDependencySet | null,
2316
+ homes: readonly string[],
2317
+ deadlineMono: number,
2318
+ pending?: PendingDelegation
2319
+ ): Promise<LockDependencySet> {
2320
+ if (pending?.recalled) throw PENDING_DELEGATION_RECALLED;
2321
+ if (requirement?.length === 0) return requirement;
2322
+ const remaining = deadlineMono - this.#monotonic();
2323
+ if (remaining <= 0) throw new Error('the lock wait elapsed before its freshness barrier started');
2324
+ const established = Promise.resolve().then(() =>
2325
+ this.transport.establishLockFreshness(this.database, this.table, key, requirement, remaining)
2326
+ );
2327
+ const timeout = delay(remaining);
2328
+ established.then(
2329
+ () => timeout.cancel(),
2330
+ () => timeout.cancel()
2331
+ );
2332
+ const alternatives: Promise<LockDependencySet | void>[] = [
2333
+ established,
2334
+ timeout.promise.then(() => {
2335
+ throw new Error('the lock wait elapsed before its freshness barrier completed');
2336
+ }),
2337
+ ];
2338
+ if (pending)
2339
+ alternatives.push(
2340
+ pending.recalledPromise.then(() => {
2341
+ timeout.cancel();
2342
+ throw PENDING_DELEGATION_RECALLED;
2343
+ })
2344
+ );
2345
+ const result = await Promise.race(alternatives);
2346
+ if (requirement !== null) return requirement;
2347
+ const recovered = normalizeDependencies(result);
2348
+ if (!recovered) throw new Error('the recovery barrier returned no usable applied-position set');
2349
+ const allowedOrigins = new Set(homes);
2350
+ return recovered.filter(([origin]) => allowedOrigins.has(origin));
2351
+ }
2352
+
2353
+ /** This node is the key's home: grant to itself through exactly the same table a peer would use. */
2354
+ #grantLocally(keyId: unknown, key: any, homeMap: LockHomeMap, leaseMs: number): DelegationReply {
2355
+ return this.#grant(keyId, key, homeMap, leaseMs, this.nodeId);
2356
+ }
2357
+
2358
+ async #requestRemotely(
2359
+ home: string,
2360
+ keyId: unknown,
2361
+ key: any,
2362
+ homeMap: LockHomeMap,
2363
+ leaseMs: number,
2364
+ deadlineMono: number
2365
+ ): Promise<DelegationReply> {
2366
+ try {
2367
+ // A half-open connection to the home would otherwise leave `lock()` pending forever, past
2368
+ // its own timeout and past the native lease — and a reply arriving after that lease has
2369
+ // fired cannot be joined to the handle anyway.
2370
+ const remaining = Math.max(1, deadlineMono - this.#monotonic());
2371
+ let raced = false;
2372
+ const requested = Promise.resolve(
2373
+ this.transport.requestDelegation(home, this.database, this.table, {
2374
+ key,
2375
+ requester: this.nodeId,
2376
+ generation: homeMap.generation,
2377
+ leaseMs,
2378
+ })
2379
+ );
2380
+ // A reply that arrives after we stopped waiting still granted us the key on the home, which
2381
+ // would then hold it for the whole delegation while every other node is denied. Hand it back.
2382
+ // Through the authority, not through this object: a transport swap can land while the reply
2383
+ // is in flight, and `handOffTo` empties this coordinator's delegations. The handback's
2384
+ // "not while a delegation for the key is held" guard would then read an empty map and give
2385
+ // back a grant that still backs the successor's live delegation.
2386
+ requested.then(
2387
+ (reply) => {
2388
+ if (raced && reply?.granted && reply.token) this.#authority().#releaseUnclaimedGrant(key, reply.token);
2389
+ },
2390
+ () => {}
2391
+ );
2392
+ const timeout = delay(remaining);
2393
+ // Dropped once the race settles: a reply that beats the timeout would otherwise leave a timer
2394
+ // holding this closure for the caller's whole remaining wait.
2395
+ requested.then(
2396
+ () => timeout.cancel(),
2397
+ () => timeout.cancel()
2398
+ );
2399
+ return await Promise.race([
2400
+ requested,
2401
+ timeout.promise.then(() => {
2402
+ raced = true;
2403
+ // `timeout`, not `contended`: the home never answered, so nobody was shown to hold the
2404
+ // key. Retried the same way, but a wait that ends here is 503 rather than 423.
2405
+ return { granted: false, reason: 'timeout', retryAfterMs: 0 } as DelegationReply;
2406
+ }),
2407
+ ]);
2408
+ } catch (error) {
2409
+ // An unreachable home blocks only the keys it homes, which is the availability property the
2410
+ // whole design exists for — it is not a reason to admit without one.
2411
+ throw new LockUnavailableError(
2412
+ `Could not reach ${home}, the home node for this key on ${this.database}.${this.table}: ${(error as Error)?.message ?? error}`
2413
+ );
2414
+ }
2415
+ }
2416
+
2417
+ /**
2418
+ * Give back a delegation this node asked for but stopped waiting on. Without it the home holds the
2419
+ * key for a delegation nobody is using, and every other node is denied for its full duration.
2420
+ */
2421
+ #releaseUnclaimedGrant(key: any, token: FencingToken): Promise<void> | void {
2422
+ const keyId = this.#keyIdOf(key);
2423
+ const held = this.#delegations.get(keyId);
2424
+ // ANY live delegation for this key means this node is using it, and the tokens need not match.
2425
+ // A duplicate or delayed request from this node renews the home's grant IN PLACE (`#grant`'s
2426
+ // renewal branch mutates `existing.token`), so the home can hold a newer token than the one we
2427
+ // installed. Releasing that token would clear the grant still backing our own live delegation
2428
+ // and let the home hand the key to another node while we are inside it. Comparing tokens here
2429
+ // caught only the case where we installed this exact grant.
2430
+ if (held) return;
2431
+ return this.#writeControlSafely({ type: 'lockRelease', key, requester: this.nodeId, token, dependencies: null });
2432
+ }
2433
+
2434
+ #grant(keyId: unknown, key: any, homeMap: LockHomeMap, leaseMs: number, requester: string): DelegationReply {
2435
+ const now = this.#monotonic();
2436
+ this.#prepareFreshnessGeneration(homeMap.generation);
2437
+ // Consulted unconditionally — it is what observes a gap in ownership, and the waiver it applies
2438
+ // to itself is the only part `grantableAfterMono` may switch off.
2439
+ const quarantine = Math.max(this.#grantableAfterMono, this.#ownershipHorizon(now, homeMap.homeIncarnation)) - now;
2440
+ // NOT `contended`: the quarantine runs for a full delegation lease, and `MAX_LOCK_TIMEOUT_MS` is
2441
+ // shorter than that, so retrying it would spend the caller's whole budget and then answer 423 —
2442
+ // "held by someone else" — for a key nobody holds.
2443
+ if (quarantine > 0) return { granted: false, reason: 'quarantine', retryAfterMs: Math.min(quarantine, 250) };
2444
+ const existing = this.#grants.get(keyId);
2445
+ if (existing) {
2446
+ // An expired grant is not a live one. Collecting it here rather than trusting `tick()` is
2447
+ // what keeps a table whose expiry budget is saturated from answering `contended` forever.
2448
+ if (existing.expiresMono <= now) this.#expireGrant(keyId, existing);
2449
+ else if (existing.recalling || existing.recallConfirmed)
2450
+ // A recall is in flight, or the delegate has already confirmed one. Renewing now — even for
2451
+ // the node being recalled — would mint a token its own release no longer matches, and the
2452
+ // contender would never get the key. `recallConfirmed` has to be here as well as in
2453
+ // `#beginRecall`: the delegate can confirm and re-ask before its release reaches the home,
2454
+ // and renewing then would leave a grant nothing will recall again and nothing can release.
2455
+ return { granted: false, reason: 'contended', retryAfterMs: 25 };
2456
+ else if (existing.delegate === requester) {
2457
+ // Renewal for the node that already holds it: extend rather than recall itself.
2458
+ this.#recordGenerationActedOn(homeMap.generation);
2459
+ existing.token = [homeMap.generation, homeMap.homeIncarnation, ++this.#counter];
2460
+ existing.renewed = true;
2461
+ existing.expiresMono = now + DELEGATION_LEASE_MS + this.#skewMs;
2462
+ return {
2463
+ granted: true,
2464
+ token: existing.token,
2465
+ leaseMs: DELEGATION_LEASE_MS,
2466
+ dependencies: existing.dependencies,
2467
+ };
2468
+ } else {
2469
+ // Someone else holds it. Start the recall and make the caller come back — holding the
2470
+ // request open across a drain would tie the home's reply to the previous delegate's
2471
+ // liveness.
2472
+ this.#beginRecall(keyId, existing);
2473
+ return { granted: false, reason: 'contended', retryAfterMs: 25 };
2474
+ }
2475
+ }
2476
+ if (this.#grants.size >= MAX_DELEGATIONS_PER_TABLE) return { granted: false, reason: 'capacity' };
2477
+ const perRequester = this.#grantsByRequester.get(requester) ?? 0;
2478
+ if (perRequester >= MAX_DELEGATIONS_PER_REQUESTER) return { granted: false, reason: 'capacity' };
2479
+ this.#recordGenerationActedOn(homeMap.generation);
2480
+ const token: FencingToken = [homeMap.generation, homeMap.homeIncarnation, ++this.#counter];
2481
+ const dependencies = this.#freshnessFor(keyId);
2482
+ this.#everDelegated.add(keyId);
2483
+ this.#grants.set(keyId, {
2484
+ key,
2485
+ delegate: requester,
2486
+ token,
2487
+ dependencies,
2488
+ // The home always outwaits the delegate by skew, so it cannot re-grant a key the previous
2489
+ // delegate still believes it holds. The delegation runs for its own fixed duration rather
2490
+ // than the caller's lock lease — a delegation sized to one lock leaves no room for the next
2491
+ // one, and every repeat lock would pay a round trip.
2492
+ expiresMono: now + DELEGATION_LEASE_MS + this.#skewMs,
2493
+ });
2494
+ this.#grantsByRequester.set(requester, perRequester + 1);
2495
+ this.#startTicking();
2496
+ return { granted: true, token, leaseMs: DELEGATION_LEASE_MS, dependencies };
2497
+ }
2498
+
2499
+ #beginRecall(keyId: unknown, grant: HomeGrant): void {
2500
+ if (grant.recalling || grant.recallConfirmed) return;
2501
+ if (grant.recallRetryAfterMono !== undefined && this.#monotonic() < grant.recallRetryAfterMono) return;
2502
+ if (grant.delegate === this.nodeId) {
2503
+ // We are both home and delegate. Recall ourselves through the same path a peer would take,
2504
+ // and settle it the same way: a recall that did not apply must leave the grant recallable,
2505
+ // or `recalling` stays latched here for the rest of the delegation and no contender ever
2506
+ // prompts another one.
2507
+ grant.recalling = this.onDelegationRecall({ key: grant.key, token: grant.token }).then(
2508
+ () => {
2509
+ grant.recalling = undefined;
2510
+ grant.recallConfirmed = true;
2511
+ },
2512
+ (error) => {
2513
+ warnOnce('failed to recall a local record lock delegation', error);
2514
+ grant.recalling = undefined;
2515
+ grant.recallRetryAfterMono = this.#monotonic() + RECALL_RETRY_MS;
2516
+ }
2517
+ );
2518
+ return;
2519
+ }
2520
+ // `.then`, not `Promise.resolve(recallDelegation(...))`: the transport call is evaluated before
2521
+ // `Promise.resolve` and can throw synchronously, which escapes this whole method — so the
2522
+ // handlers below never run, the retry interval is never armed, and the next contender pass
2523
+ // throws again immediately instead of backing off.
2524
+ grant.recalling = Promise.resolve()
2525
+ .then(() =>
2526
+ this.transport.recallDelegation(grant.delegate, this.database, this.table, {
2527
+ key: grant.key,
2528
+ token: grant.token,
2529
+ })
2530
+ )
2531
+ .then(() => {
2532
+ // The delegate confirmed it stopped admitting, so re-sending buys nothing: the grant is
2533
+ // cleared by its release entry, or failing that by its own deadline. Re-arming here is what
2534
+ // let a contender polling at 25 ms fire a recall per pass for the rest of the delegation.
2535
+ grant.recalling = undefined;
2536
+ grant.recallConfirmed = true;
2537
+ })
2538
+ .catch(() => {
2539
+ // An unreachable delegate is not a reason to re-grant early: the grant's own deadline is
2540
+ // what makes the successor safe, and it already includes the skew margin. Retrying IS
2541
+ // worthwhile here — the delegate may come back — but on its own interval, not the
2542
+ // contender's.
2543
+ grant.recalling = undefined;
2544
+ grant.recallRetryAfterMono = this.#monotonic() + RECALL_RETRY_MS;
2545
+ });
2546
+ }
2547
+
2548
+ /**
2549
+ * Give up a delegation: stop admitting, then write the release that lets the home re-grant. One
2550
+ * surrender per delegation, memoized: the home re-sends a recall `RECALL_RETRY_MS` after its own
2551
+ * recall call failed, which is far inside a delegation lease, so two recalls for the same token can
2552
+ * both be waiting on the drain and resume together. Without the memo the second finds the
2553
+ * admissions the first already emptied, has no fence to wait for, and writes the release while the
2554
+ * first is still waiting for a relayed handle to confirm it is fenced.
2555
+ */
2556
+ #surrender(keyId: unknown, delegation: Delegation): Promise<void> {
2557
+ return (delegation.surrendering ??= this.#surrenderOnce(keyId, delegation));
2558
+ }
2559
+
2560
+ async #surrenderOnce(keyId: unknown, delegation: Delegation): Promise<void> {
2561
+ if (this.#delegations.get(keyId) === delegation) this.#delegations.delete(keyId);
2562
+ // Capability, not just admission: anything this delegation admitted must be unable to commit
2563
+ // before the home is told it may re-grant. A handle admitted on another worker thread revokes
2564
+ // over a message, so wait for its fence to land — bounded by the handle's own lease, past which
2565
+ // it fences itself — before writing the release that lets the home re-grant the key.
2566
+ await this.#revokeAllAndSettle(delegation);
2567
+ const entry: LockReleaseEntry = {
2568
+ type: 'lockRelease',
2569
+ key: delegation.key,
2570
+ requester: this.nodeId,
2571
+ token: delegation.token,
2572
+ dependencies: delegation.dependencies,
2573
+ };
2574
+ return this.#writeControlSafely(entry);
2575
+ }
2576
+
2577
+ /**
2578
+ * Revoke every handle this delegation is answerable for and forget the admissions. Returns, per
2579
+ * admission, the revoker's outcome paired with the admission's own monotonic deadline, so a caller
2580
+ * that must not write the release before the fence is proven (`#surrender`) can bound its wait by
2581
+ * that deadline. Callers tearing the delegation down anyway (`tick`, `close`) ignore the result. A
2582
+ * revoke that throws synchronously is captured as a rejected outcome, never swallowed as fenced.
2583
+ */
2584
+ #revokeAll(delegation: Delegation): { outcome: void | Promise<void>; expiresMono: number }[] {
2585
+ const admissions = delegation.admissions;
2586
+ delegation.admissions = new Map();
2587
+ delegation.holding = 0;
2588
+ delegation.sweepAtSize = ADMISSION_SWEEP_FLOOR;
2589
+ const outcomes: { outcome: void | Promise<void>; expiresMono: number }[] = [];
2590
+ for (const [admissionId, admission] of admissions) {
2591
+ this.#admissions.delete(admissionId);
2592
+ let outcome: void | Promise<void>;
2593
+ try {
2594
+ outcome = admission.revoke();
2595
+ } catch (error) {
2596
+ // A synchronous throw is NOT a fence: surface it as a rejected outcome so `#revokeAllAndSettle`
2597
+ // waits out the admission's own lease rather than releasing against a handle that may still commit.
2598
+ outcome = Promise.reject(error);
2599
+ }
2600
+ // Every fire-and-forget caller (`tick`, `close`) discards this array, so a rejecting outcome —
2601
+ // a sync throw above, or a relayed revoker whose promise rejects on a dead sibling port — must
2602
+ // carry its own no-op handler here or Node's default policy would exit the worker. The awaited
2603
+ // path (`#revokeAllAndSettle`) still sees the rejection: a settled promise may be awaited again.
2604
+ if (outcome && typeof (outcome as Promise<void>).then === 'function') (outcome as Promise<void>).catch(() => {});
2605
+ outcomes.push({ outcome, expiresMono: admission.expiresMono });
2606
+ }
2607
+ return outcomes;
2608
+ }
2609
+
2610
+ /**
2611
+ * `#revokeAll`, then wait for every asynchronous fence (a handle on another worker acking its
2612
+ * `revokeLease`) before the caller writes the release. Each wait is bounded HERE by the admission's
2613
+ * own remaining lease, independent of what the transport's revoker does: a revoker that rejects,
2614
+ * throws, or never settles cannot make `#surrender` publish the release before the fenced handle's
2615
+ * lease has elapsed — past which the handle fences itself and can no longer commit. The transport's
2616
+ * own lease-bounded ack (harper-pro) is the fast path; this is the guarantee.
2617
+ */
2618
+ async #revokeAllAndSettle(delegation: Delegation): Promise<void> {
2619
+ const outcomes = this.#revokeAll(delegation);
2620
+ // ONE shared lease timer for the whole settle, not one per admission: a recall of a delegation
2621
+ // holding thousands of relayed admissions would otherwise arm thousands of timers at exactly the
2622
+ // moment it is handing off. The shared deadline is the LATEST lease among them, which bounds every
2623
+ // handle in the set — waiting past a shorter lease only ever errs towards holding the release
2624
+ // longer, never towards writing it early.
2625
+ let latestExpiry = -Infinity;
2626
+ let anyAsync = false;
2627
+ for (const { outcome, expiresMono } of outcomes) {
2628
+ if (!isPromiseLike(outcome)) continue;
2629
+ anyAsync = true;
2630
+ if (expiresMono > latestExpiry) latestExpiry = expiresMono;
2631
+ }
2632
+ // A synchronous revoker has already fenced by the time it returns, so it contributes nothing to
2633
+ // wait on: only the promise-returning ones cost an entry here.
2634
+ if (!anyAsync) return;
2635
+ const leaseTimer = delay(Math.max(0, latestExpiry - this.#monotonic()));
2636
+ const waits: Promise<unknown>[] = [];
2637
+ for (const { outcome } of outcomes) {
2638
+ if (!isPromiseLike(outcome)) continue;
2639
+ // Race the fence ack against the shared lease deadline: whichever comes first, the handle can no
2640
+ // longer commit once we return. A rejected fence is not a confirmed one, so it falls through to
2641
+ // the same deadline rather than resolving early.
2642
+ waits.push(
2643
+ Promise.race([
2644
+ outcome.catch((error) => {
2645
+ warnOnce('a record lock handle did not confirm revocation; waiting out its lease', error);
2646
+ return leaseTimer.promise;
2647
+ }),
2648
+ leaseTimer.promise,
2649
+ ])
2650
+ );
2651
+ }
2652
+ try {
2653
+ await Promise.all(waits);
2654
+ } finally {
2655
+ leaseTimer.cancel();
2656
+ }
2657
+ }
2658
+
2659
+ #clearGrant(keyId: unknown, grant: HomeGrant): void {
2660
+ if (this.#grants.get(keyId) !== grant) return;
2661
+ this.#grants.delete(keyId);
2662
+ const count = (this.#grantsByRequester.get(grant.delegate) ?? 1) - 1;
2663
+ if (count > 0) this.#grantsByRequester.set(grant.delegate, count);
2664
+ else this.#grantsByRequester.delete(grant.delegate);
2665
+ }
2666
+
2667
+ #expireGrant(keyId: unknown, grant: HomeGrant): void {
2668
+ if (this.#grants.get(keyId) !== grant) return;
2669
+ this.#dependencySets.delete(keyId);
2670
+ this.#clearGrant(keyId, grant);
2671
+ }
2672
+
2673
+ async #writeControlSafely(entry: LockReleaseEntry): Promise<void> {
2674
+ try {
2675
+ const position = await this.#writeControl(entry);
2676
+ this.#authority().applyEntry(entry, this.nodeId, typeof position === 'number' ? position : undefined);
2677
+ } catch (error) {
2678
+ // A lost release costs the key its remaining lease on the home; it never costs exclusion.
2679
+ warnOnce('failed to write a record lock release entry', error);
2680
+ // A local home can discard its own failed handback safely; its successor will recover.
2681
+ try {
2682
+ const keyId = this.#keyIdOf(entry.key);
2683
+ const grant = this.#grants.get(keyId);
2684
+ if (grant?.delegate === this.nodeId && compareTokens(grant.token, entry.token) === 0)
2685
+ this.#expireGrant(keyId, grant);
2686
+ } catch {}
2687
+ }
2688
+ }
2689
+
2690
+ #startTicking() {
2691
+ if (!this.#autoTick) return;
2692
+ tickingCoordinators.add(this);
2693
+ ensureTicking();
2694
+ }
2695
+ }
2696
+
2697
+ /** A sleep whose timer the winner of a race can drop, rather than let it run out its full delay. */
2698
+ function delay(ms: number): { promise: Promise<void>; cancel: () => void } {
2699
+ let timer: ReturnType<typeof setTimeout>;
2700
+ const promise = new Promise<void>((resolve) => {
2701
+ timer = setTimeout(resolve, ms);
2702
+ timer.unref?.();
2703
+ });
2704
+ return { promise, cancel: () => clearTimeout(timer!) };
2705
+ }
2706
+
2707
+ const clusterLockTransports = new Map<string, ClusterLockTransport>();
2708
+ // Databases that have had a transport registered on this worker. A later absence is a transport
2709
+ // that went away — a component reload, a failed reconnect — not proof the node became standalone,
2710
+ // so cluster scope must keep failing closed rather than quietly reverting to a node-local lock.
2711
+ const clusterRequiredDatabases = new Set<string>();
2712
+ type CoordinatorResolver = (database: string, table: string) => LockCoordinator | undefined;
2713
+ let coordinatorResolver: CoordinatorResolver | undefined;
2714
+ // Applying a release is bookkeeping on this node's own grant table and needs no transport, so it
2715
+ // resolves the coordinator that HOLDS the grant rather than the one a registered transport answers
2716
+ // for. Dropping a peer's clean-handoff release during a reconnect denies every other node that key
2717
+ // for the delegation's whole deadline. Requests and recalls keep the transport-gated resolver: both
2718
+ // are answers to a live peer, and failing them closed while the transport is gone is the right shape.
2719
+ let admittingResolver: CoordinatorResolver | undefined;
2720
+ type ControlWriter = (entry: LockControlEntry) => Promise<number | void> | number | void;
2721
+ type ControlWriterResolver = (database: string, table: string) => ControlWriter | undefined;
2722
+ let controlWriterResolver: ControlWriterResolver | undefined;
2723
+
2724
+ /** Installed by Table.ts so a transport can push received entries in without importing Table. */
2725
+ export function setLockCoordinatorResolver(
2726
+ resolve: CoordinatorResolver,
2727
+ resolveAdmitting: CoordinatorResolver = resolve,
2728
+ resolveControlWriter?: ControlWriterResolver
2729
+ ) {
2730
+ coordinatorResolver = resolve;
2731
+ admittingResolver = resolveAdmitting;
2732
+ controlWriterResolver = resolveControlWriter;
2733
+ }
2734
+
2735
+ export interface QuiesceOutstanding {
2736
+ table: string;
2737
+ key: unknown;
2738
+ /** Present when this node was the HOME and the delegate did not confirm. */
2739
+ delegate?: string;
2740
+ reason: string;
2741
+ }
2742
+
2743
+ export interface QuiesceResult {
2744
+ /**
2745
+ * Whether this result is a PROOF of quiescence, or merely a report of what was swept.
2746
+ *
2747
+ * A sweep can only visit coordinators that exist on this thread, and they are built lazily — a
2748
+ * table nothing has touched since a restart has none, so an empty `outstanding` would otherwise
2749
+ * read as "nothing is live" when the previous incarnation's delegations are still running
2750
+ * elsewhere. An orchestrator must require `complete && outstanding.length === 0`; anything else
2751
+ * means fall back to the drain interval for this node.
2752
+ */
2753
+ complete: boolean;
2754
+ /** Delegations this node held and gave up, so it can no longer admit under them. */
2755
+ surrendered: number;
2756
+ /** Grants this node issued whose delegate confirmed it stopped admitting. */
2757
+ recalled: number;
2758
+ /**
2759
+ * What is still live, or unprovable. Empty means this node is provably quiesced for the database —
2760
+ * and ONLY then, which is why a coordinator still inside its restart quarantine contributes an
2761
+ * entry here rather than reporting a clean sweep it cannot back.
2762
+ */
2763
+ outstanding: QuiesceOutstanding[];
2764
+ }
2765
+
2766
+ /**
2767
+ * Stop this node admitting under the current generation for `database`, and say whether it is
2768
+ * provably done (harper-pro#856).
2769
+ *
2770
+ * A membership change otherwise has to wait out `DELEGATION_LEASE_MS + LOCK_LEASE_SKEW_MS` before the
2771
+ * next generation may activate, because authority already issued under the old one has to expire —
2772
+ * roughly six minutes during which the staged nodes serve no cluster locks at all. That interval is a
2773
+ * TIMER, chosen because you cannot recall what you cannot reach. In a planned transition every
2774
+ * participant is reachable, so the same guarantee can be *established* instead of waited out: this
2775
+ * drains both directions and reports what, if anything, is left.
2776
+ *
2777
+ * - **As a delegate** it surrenders every delegation it holds. This is the load-bearing half: a
2778
+ * delegate is what admits, and `onDelegationRecall` is the existing path whose resolution means
2779
+ * "nothing on this node can admit on that token" — live critical sections are drained, not cut.
2780
+ * - **As a home** it recalls every grant it issued, so delegates elsewhere stop too. Redundant when
2781
+ * every node is quiescing at once, and the reason this still terminates when one is not.
2782
+ *
2783
+ * `outstanding` empty on every node in `homes(g) ∪ homes(g+1)` is the operator's evidence that the
2784
+ * next generation may be activated immediately. Anything left is a node to fall back to the timer for,
2785
+ * or to fence externally — this never claims a drain it did not get, and never throws for one grant.
2786
+ */
2787
+ export async function quiesceDelegations(database: string, budgetMs: number): Promise<QuiesceResult> {
2788
+ const result: QuiesceResult = { complete: true, surrendered: 0, recalled: 0, outstanding: [] };
2789
+ const coordinators = [...liveCoordinators].filter((coordinator) => coordinator.database === database);
2790
+ // A DURATION, not an absolute deadline: the whole sweep gets this long, measured from here.
2791
+ const deadline = Date.now() + Math.max(0, budgetMs);
2792
+ const remaining = () => Math.max(0, deadline - Date.now());
2793
+ for (const coordinator of coordinators) {
2794
+ await coordinator.quiesce(result, remaining);
2795
+ }
2796
+ // A sweep proves quiescence only if it could have seen everything this NODE issued, not merely what
2797
+ // this coordinator holds. Two ways it could not have:
2798
+ //
2799
+ // - No coordinator exists for the database on this thread, so there is nothing to attest from.
2800
+ // - A coordinator has not owned coordination long enough for authority issued BEFORE it took over
2801
+ // — by a previous owner thread, or a previous incarnation of this process — to have expired.
2802
+ // Those grants live on delegate nodes and no local sweep can see them; the same fact is what
2803
+ // core's own grant gate refuses on, and it is why uptime is not the right measure (a worker that
2804
+ // built empty coordinators early and took ownership late has plenty of uptime and no proof).
2805
+ if (coordinators.length === 0) {
2806
+ result.outstanding.push({
2807
+ table: '*',
2808
+ key: undefined,
2809
+ reason:
2810
+ 'no lock coordinator exists for this database on this thread, so there is nothing to prove quiescence from',
2811
+ });
2812
+ }
2813
+ for (const coordinator of coordinators) {
2814
+ const unproven = coordinator.unprovenOwnershipMs();
2815
+ if (unproven > 0)
2816
+ result.outstanding.push({
2817
+ table: coordinator.table,
2818
+ key: undefined,
2819
+ reason: `this thread has not coordinated ${database}.${coordinator.table} long enough to rule out authority issued before it took over; ${Math.ceil(unproven)}ms remain`,
2820
+ });
2821
+ }
2822
+ // A coordinator that closed parked the latest deadline of the grants it had issued to OTHER nodes
2823
+ // here and then cleared its own table (`close`). Those grants are still valid on their delegates and
2824
+ // no live coordinator holds them, so a sweep that ignored this would miss them entirely — the table
2825
+ // may not even have a coordinator any more.
2826
+ //
2827
+ // `performance.now()` because a retired entry outlives the coordinator whose injected clock produced
2828
+ // its deadline: production passes that same clock (the transport's `monotonicNow`), so the domains
2829
+ // agree where it matters, and a test on an artificial clock only ever reads the deadline as further
2830
+ // away — conservative, never a false clean.
2831
+ const now = performance.now();
2832
+ for (const [key, retired] of retiredCoordinators) {
2833
+ const separator = key.indexOf('\u0000');
2834
+ if (separator < 0 || key.slice(0, separator) !== database) continue;
2835
+ if (!(retired.grantableAfterMono > now)) continue;
2836
+ result.outstanding.push({
2837
+ table: key.slice(separator + 1),
2838
+ key: undefined,
2839
+ reason: `a closed coordinator for this table issued grants that remain valid on their delegates for another ${Math.ceil(retired.grantableAfterMono - now)}ms`,
2840
+ });
2841
+ }
2842
+ if (result.outstanding.length > 0) result.complete = false;
2843
+ return result;
2844
+ }
2845
+
2846
+ /**
2847
+ * Commit a `lockBarrier` entry for the table and resolve to its transaction-log position — the §7.2
2848
+ * recovery fence, for a transport answering a peer's recovery probe. The entry is appended after
2849
+ * every transaction this node had committed when the call was made, so a peer that has applied this
2850
+ * origin's log through the returned position has applied all of them.
2851
+ *
2852
+ * The transport supplies the nonce it will match the entry on, since a position alone is not an
2853
+ * identity: a restart after the wall clock moved backwards can reissue a log key, and a drain that
2854
+ * matched the earlier entry at that key would declare this origin drained with its post-restart
2855
+ * commits unapplied.
2856
+ *
2857
+ * Strictly this node's own commit, never the transport's `writeControl`: the fence is a position in
2858
+ * THIS origin's log, and the caller is the transport itself — a relaying hook would answer with a
2859
+ * position that is not local, or re-enter the operation that called here. A write that commits
2860
+ * without a position rejects rather than resolve, since a barrier nobody can wait on is not a fence.
2861
+ */
2862
+ export async function writeLockBarrier(database: string, table: string, nonce: number): Promise<number> {
2863
+ if (!Number.isSafeInteger(nonce) || nonce < 0)
2864
+ throw new ClientError('A lock barrier nonce must be a non-negative integer');
2865
+ const write = controlWriterResolver?.(database, table);
2866
+ if (!write) throw new ClientError(`Table ${database}.${table} does not exist`, 404);
2867
+ const position = await write({ type: 'lockBarrier', nonce });
2868
+ if (typeof position !== 'number' || !(position >= 0) || !Number.isFinite(position))
2869
+ throw new LockUnavailableError(`the record lock barrier for ${database}.${table} committed without a log position`);
2870
+ return position;
2871
+ }
2872
+
2873
+ /**
2874
+ * Register harper-pro's transport for a database, on THIS thread.
2875
+ *
2876
+ * **It must be registered on every worker that can serve a `lock()`, not only the coordinating one**
2877
+ * — including a dedicated application worker (harper#2524) — and core cannot check that. `clusterRequiredDatabases` is module state, so a worker that never registers never latches —
2878
+ * and a default-scoped `lock()` there takes the Phase 0 node lock alone while a peer runs the cluster
2879
+ * protocol, which is two nodes admitting one key. The `ownsCoordination()` fail-closed path only
2880
+ * reaches a worker that has a transport. Registering everywhere also makes that path the one a
2881
+ * non-owner worker takes, which is what it exists for.
2882
+ */
2883
+ export function registerClusterLockTransport(database: string, transport: ClusterLockTransport): void {
2884
+ if (
2885
+ typeof transport?.homeMap !== 'function' ||
2886
+ typeof transport?.ownsCoordination !== 'function' ||
2887
+ typeof transport?.requestDelegation !== 'function' ||
2888
+ typeof transport?.recallDelegation !== 'function' ||
2889
+ typeof transport?.establishLockFreshness !== 'function'
2890
+ )
2891
+ throw new ClientError(
2892
+ 'A cluster lock transport must provide homeMap(), ownsCoordination(), requestDelegation(), recallDelegation() and establishLockFreshness()'
2893
+ );
2894
+ transport.onControlEntry = (db: string, table: string, entry: LockControlEntry, author: string, position: number) =>
2895
+ deliverLockControlEntry(db, table, entry, author, position);
2896
+ transport.onDelegationRequest = (db: string, table: string, request: DelegationRequest) =>
2897
+ deliverDelegationRequest(db, table, request);
2898
+ transport.onDelegationRecall = (db: string, table: string, recall: DelegationRecall) =>
2899
+ deliverDelegationRecall(db, table, recall);
2900
+ clusterRequiredDatabases.add(database);
2901
+ clusterLockTransports.set(database, transport);
2902
+ }
2903
+
2904
+ export function unregisterClusterLockTransport(database: string, standalone = false): void {
2905
+ clusterLockTransports.delete(database);
2906
+ // Only an explicit statement that the database is no longer clustered clears the requirement.
2907
+ if (standalone) clusterRequiredDatabases.delete(database);
2908
+ }
2909
+
2910
+ /** True once a transport has been registered for this database and no standalone claim has cleared it. */
2911
+ export function isClusterLockRequired(database: string): boolean {
2912
+ return clusterRequiredDatabases.size > 0 && clusterRequiredDatabases.has(database);
2913
+ }
2914
+
2915
+ /**
2916
+ * The registered transport, if any. The `size` check keeps the Phase 0 path free of a map lookup on
2917
+ * every `lock()` in a build where no transport is ever registered.
2918
+ */
2919
+ export function getClusterLockTransport(database: string): ClusterLockTransport | undefined {
2920
+ if (clusterLockTransports.size === 0) return undefined;
2921
+ return clusterLockTransports.get(database);
2922
+ }
2923
+
2924
+ export function hasClusterLockTransports(): boolean {
2925
+ return clusterLockTransports.size > 0;
2926
+ }
2927
+
2928
+ /**
2929
+ * Resolve a coordinator for an inbound message. The resolver reaches `Table.lockCoordinator`, which
2930
+ * throws when this node's name is unusable — that throw must not escape a receive boundary, or it
2931
+ * reaches the replicated apply loop and drops the enclosing transaction.
2932
+ */
2933
+ function coordinatorFor(database: string, table: string, resolve = coordinatorResolver): LockCoordinator | undefined {
2934
+ try {
2935
+ return resolve?.(database, table);
2936
+ } catch (error) {
2937
+ warnOnce('could not resolve a record lock coordinator for a received message', error);
2938
+ return undefined;
2939
+ }
2940
+ }
2941
+
2942
+ export function deliverLockControlEntry(
2943
+ database: string,
2944
+ table: string,
2945
+ entry: LockControlEntry,
2946
+ author: string,
2947
+ position: number
2948
+ ): void {
2949
+ coordinatorFor(database, table, admittingResolver)?.applyEntry(entry, author, position);
2950
+ }
2951
+
2952
+ export async function deliverDelegationRequest(
2953
+ database: string,
2954
+ table: string,
2955
+ request: DelegationRequest
2956
+ ): Promise<DelegationReply> {
2957
+ const coordinator = coordinatorFor(database, table);
2958
+ if (!coordinator) return { granted: false, reason: 'not-home' };
2959
+ try {
2960
+ return await coordinator.onDelegationRequest(request);
2961
+ } catch (error) {
2962
+ warnOnce('failed to answer a record lock delegation request', error);
2963
+ return { granted: false, reason: 'not-home' };
2964
+ }
2965
+ }
2966
+
2967
+ export async function deliverDelegationRecall(
2968
+ database: string,
2969
+ table: string,
2970
+ recall: DelegationRecall
2971
+ ): Promise<void> {
2972
+ // This is core's receiving end of `recallDelegation`, which resolves only "once the delegate has
2973
+ // drained and stopped admitting" — and `#beginRecall` latches `recallConfirmed` on that resolution
2974
+ // and never re-sends. So a recall this thread could not apply has to FAIL rather than resolve:
2975
+ // `coordinatorFor` is the transport-gated resolver and answers undefined through a reconnect, and
2976
+ // the home reading that silence as a drained delegate denies the key to every other node for the
2977
+ // delegation's whole deadline. Failing it leaves the home's `.catch` to retry on RECALL_RETRY_MS.
2978
+ const coordinator = coordinatorFor(database, table);
2979
+ if (!coordinator)
2980
+ throw new Error(`No record lock coordinator on this thread to apply a recall for ${database}.${table}`);
2981
+ await coordinator.onDelegationRecall(recall);
2982
+ }
2983
+
2984
+ // ---- owner-worker relay (harper-pro#852) -------------------------------------------------------
2985
+
2986
+ /**
2987
+ * The owner thread's end of `acquireOnOwner`: mint an admission for a `lock()` served on another
2988
+ * worker thread, and register `revoke` as the way to fence that thread's handle. `revoke` is what the
2989
+ * transport wires to a cross-thread message; a recall or surrender here calls it and waits for the
2990
+ * fence before writing the release. Runs on the coordinating thread, resolved through the transport-
2991
+ * gated resolver so it fails when this thread does not coordinate the database — the same shape as
2992
+ * `deliverDelegationRequest`. Returns the round the calling worker installs as a remote admission.
2993
+ */
2994
+ export async function acquireForRelay(
2995
+ database: string,
2996
+ table: string,
2997
+ key: any,
2998
+ leaseMs: number,
2999
+ waitMs: number,
3000
+ makeRevoke: (round: LockRound) => () => void | Promise<void>
3001
+ ): Promise<LockRound> {
3002
+ const coordinator = coordinatorFor(database, table);
3003
+ if (!coordinator)
3004
+ throw new Error(`No record lock coordinator on this thread to acquire ${database}.${table} for a peer worker`);
3005
+ const round = await coordinator.acquire(key, leaseMs, waitMs);
3006
+ // Re-resolve through the ADMITTING resolver rather than reusing the captured coordinator: a transport
3007
+ // swap during the acquire moves the admission to the successor and empties the predecessor, so
3008
+ // registering on the captured object would eagerly revoke a healthy handle and leave the successor's
3009
+ // admission unfenceable. `Table.ts` does the same via `admittingCoordinator`. The revoker is built
3010
+ // from the round so it names the exact admission when it tells the calling worker to fence its handle.
3011
+ const authority = coordinatorFor(database, table, admittingResolver) ?? coordinator;
3012
+ authority.registerAdmission(round.admissionId, makeRevoke(round));
3013
+ return round;
3014
+ }
3015
+
3016
+ /** The owner thread's end of `releaseOnOwner`: end a relayed admission the owner minted. */
3017
+ export function releaseForRelay(database: string, table: string, key: any, admissionId: number): Promise<void> | void {
3018
+ // The admitting resolver, like a received release: it answers the coordinator that HOLDS the
3019
+ // admission even while a transport is momentarily unregistered, so a release is never dropped.
3020
+ return coordinatorFor(database, table, admittingResolver)?.release(key, admissionId);
3021
+ }
3022
+
3023
+ /**
3024
+ * The calling thread's end of an owner `revoke`: fence the handle for a relayed admission this thread
3025
+ * holds. Resolves once the handle's `revokeLease` has run, so the owner may wait for the fence before
3026
+ * it writes the release. The admitting resolver answers the coordinator that adopted the admission
3027
+ * across a transport swap.
3028
+ */
3029
+ export function revokeRelayedAdmission(database: string, table: string, admissionId: number): Promise<void> {
3030
+ const coordinator = coordinatorFor(database, table, admittingResolver);
3031
+ // No coordinator to fence against means nothing here can commit under that admission; the fence is
3032
+ // vacuously satisfied and the owner may proceed.
3033
+ return coordinator ? coordinator.revokeRemoteAdmission(admissionId) : Promise.resolve();
3034
+ }
3035
+
3036
+ /**
3037
+ * Fail-closed fence for every relayed handle a table's coordinator holds (harper-pro#852), for when
3038
+ * the owner worker that granted them has exited and its delegations are gone. Harper-pro calls this
3039
+ * per table when it learns the coordinating thread for a database changed.
3040
+ */
3041
+ export function fenceRelayedAdmissions(database: string, table: string): void {
3042
+ coordinatorFor(database, table, admittingResolver)?.fenceAllRemoteAdmissions();
3043
+ }