@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/bin/cliOperations.ts +12 -0
  2. package/bin/restart.ts +66 -6
  3. package/components/Application.ts +1134 -112
  4. package/components/DESIGN.md +421 -0
  5. package/components/OptionsWatcher.ts +368 -102
  6. package/components/Scope.ts +20 -1
  7. package/components/componentLoader.ts +43 -4
  8. package/components/deploymentOperations.ts +4 -1
  9. package/components/deploymentRecorder.ts +9 -2
  10. package/components/mcp/DESIGN.md +109 -0
  11. package/components/mcp/audit.ts +21 -17
  12. package/components/operations.js +284 -52
  13. package/components/operationsValidation.js +49 -2
  14. package/components/packageComponent.ts +25 -1
  15. package/components/requestRestart.ts +11 -0
  16. package/config/DESIGN.md +306 -0
  17. package/config/RootConfigWatcher.ts +191 -37
  18. package/config/configReadRetry.ts +62 -0
  19. package/config/configUtils.ts +78 -26
  20. package/config/parseConfigFile.ts +34 -0
  21. package/config/readConfigFileSync.ts +44 -0
  22. package/config/watcherArming.ts +59 -0
  23. package/config-root.schema.json +4 -0
  24. package/dataLayer/DESIGN.md +179 -0
  25. package/dataLayer/harperBridge/ResourceBridge.ts +28 -2
  26. package/dataLayer/restoreMarker.ts +92 -25
  27. package/dist/bin/cliOperations.js +13 -0
  28. package/dist/bin/cliOperations.js.map +1 -1
  29. package/dist/bin/restart.js +42 -6
  30. package/dist/bin/restart.js.map +1 -1
  31. package/dist/components/Application.d.ts +104 -9
  32. package/dist/components/Application.js +954 -102
  33. package/dist/components/Application.js.map +1 -1
  34. package/dist/components/OptionsWatcher.d.ts +4 -1
  35. package/dist/components/OptionsWatcher.js +378 -104
  36. package/dist/components/OptionsWatcher.js.map +1 -1
  37. package/dist/components/Scope.js +15 -1
  38. package/dist/components/Scope.js.map +1 -1
  39. package/dist/components/componentLoader.js +35 -3
  40. package/dist/components/componentLoader.js.map +1 -1
  41. package/dist/components/deploymentOperations.js +4 -1
  42. package/dist/components/deploymentOperations.js.map +1 -1
  43. package/dist/components/deploymentRecorder.d.ts +4 -2
  44. package/dist/components/deploymentRecorder.js +1 -0
  45. package/dist/components/deploymentRecorder.js.map +1 -1
  46. package/dist/components/mcp/audit.d.ts +2 -1
  47. package/dist/components/mcp/audit.js +21 -17
  48. package/dist/components/mcp/audit.js.map +1 -1
  49. package/dist/components/operations.d.ts +28 -0
  50. package/dist/components/operations.js +263 -54
  51. package/dist/components/operations.js.map +1 -1
  52. package/dist/components/operationsValidation.js +48 -2
  53. package/dist/components/operationsValidation.js.map +1 -1
  54. package/dist/components/packageComponent.js +24 -0
  55. package/dist/components/packageComponent.js.map +1 -1
  56. package/dist/components/requestRestart.d.ts +1 -0
  57. package/dist/components/requestRestart.js +7 -0
  58. package/dist/components/requestRestart.js.map +1 -1
  59. package/dist/config/RootConfigWatcher.d.ts +2 -0
  60. package/dist/config/RootConfigWatcher.js +189 -35
  61. package/dist/config/RootConfigWatcher.js.map +1 -1
  62. package/dist/config/configReadRetry.d.ts +8 -0
  63. package/dist/config/configReadRetry.js +62 -0
  64. package/dist/config/configReadRetry.js.map +1 -0
  65. package/dist/config/configUtils.d.ts +10 -9
  66. package/dist/config/configUtils.js +63 -27
  67. package/dist/config/configUtils.js.map +1 -1
  68. package/dist/config/parseConfigFile.d.ts +4 -0
  69. package/dist/config/parseConfigFile.js +35 -0
  70. package/dist/config/parseConfigFile.js.map +1 -0
  71. package/dist/config/readConfigFileSync.d.ts +1 -0
  72. package/dist/config/readConfigFileSync.js +47 -0
  73. package/dist/config/readConfigFileSync.js.map +1 -0
  74. package/dist/config/watcherArming.d.ts +15 -0
  75. package/dist/config/watcherArming.js +59 -0
  76. package/dist/config/watcherArming.js.map +1 -0
  77. package/dist/dataLayer/harperBridge/ResourceBridge.js +21 -2
  78. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  79. package/dist/dataLayer/restoreMarker.d.ts +21 -8
  80. package/dist/dataLayer/restoreMarker.js +94 -27
  81. package/dist/dataLayer/restoreMarker.js.map +1 -1
  82. package/dist/index.d.ts +2 -0
  83. package/dist/index.js +9 -1
  84. package/dist/index.js.map +1 -1
  85. package/dist/json/systemSchema.json +3 -0
  86. package/dist/resources/DatabaseTransaction.d.ts +25 -0
  87. package/dist/resources/DatabaseTransaction.js +224 -18
  88. package/dist/resources/DatabaseTransaction.js.map +1 -1
  89. package/dist/resources/LMDBTransaction.d.ts +2 -1
  90. package/dist/resources/LMDBTransaction.js +22 -3
  91. package/dist/resources/LMDBTransaction.js.map +1 -1
  92. package/dist/resources/PrimaryRocksDatabase.js +22 -6
  93. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  94. package/dist/resources/RecordEncoder.d.ts +1 -1
  95. package/dist/resources/RecordEncoder.js +46 -10
  96. package/dist/resources/RecordEncoder.js.map +1 -1
  97. package/dist/resources/Resource.js +97 -13
  98. package/dist/resources/Resource.js.map +1 -1
  99. package/dist/resources/ResourceInterface.d.ts +8 -0
  100. package/dist/resources/RocksIndexStore.js +2 -1
  101. package/dist/resources/RocksIndexStore.js.map +1 -1
  102. package/dist/resources/RocksTransactionLogStore.d.ts +10 -0
  103. package/dist/resources/RocksTransactionLogStore.js +104 -33
  104. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  105. package/dist/resources/Table.d.ts +58 -7
  106. package/dist/resources/Table.js +1143 -348
  107. package/dist/resources/Table.js.map +1 -1
  108. package/dist/resources/analytics/write.d.ts +3 -0
  109. package/dist/resources/analytics/write.js +49 -16
  110. package/dist/resources/analytics/write.js.map +1 -1
  111. package/dist/resources/auditStore.d.ts +170 -0
  112. package/dist/resources/auditStore.js +457 -11
  113. package/dist/resources/auditStore.js.map +1 -1
  114. package/dist/resources/crdt.d.ts +10 -0
  115. package/dist/resources/crdt.js +22 -0
  116. package/dist/resources/crdt.js.map +1 -1
  117. package/dist/resources/dataLoader.js +3 -4
  118. package/dist/resources/dataLoader.js.map +1 -1
  119. package/dist/resources/databases.d.ts +16 -13
  120. package/dist/resources/databases.js +625 -177
  121. package/dist/resources/databases.js.map +1 -1
  122. package/dist/resources/derivedIndexRegistry.d.ts +5 -0
  123. package/dist/resources/derivedIndexRegistry.js +68 -0
  124. package/dist/resources/derivedIndexRegistry.js.map +1 -0
  125. package/dist/resources/derivedIndexRuntime.d.ts +215 -0
  126. package/dist/resources/derivedIndexRuntime.js +2027 -0
  127. package/dist/resources/derivedIndexRuntime.js.map +1 -0
  128. package/dist/resources/graphql.js +3 -2
  129. package/dist/resources/graphql.js.map +1 -1
  130. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +102 -9
  131. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +867 -39
  132. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  133. package/dist/resources/indexes/fullTextDerivedIndex.d.ts +81 -0
  134. package/dist/resources/indexes/fullTextDerivedIndex.js +1004 -0
  135. package/dist/resources/indexes/fullTextDerivedIndex.js.map +1 -0
  136. package/dist/resources/indexes/fullTextNativeBinding.d.ts +78 -0
  137. package/dist/resources/indexes/fullTextNativeBinding.js +85 -0
  138. package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -0
  139. package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
  140. package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
  141. package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
  142. package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
  143. package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
  144. package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
  145. package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.d.ts +24 -0
  146. package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js +149 -0
  147. package/dist/resources/indexes/nativeFullTextDerivedIndexLifecycle.js.map +1 -0
  148. package/dist/resources/nodeIdMapping.d.ts +5 -0
  149. package/dist/resources/nodeIdMapping.js +49 -0
  150. package/dist/resources/nodeIdMapping.js.map +1 -1
  151. package/dist/resources/recordLock.d.ts +47 -4
  152. package/dist/resources/recordLock.js +138 -7
  153. package/dist/resources/recordLock.js.map +1 -1
  154. package/dist/resources/recordLockCoordinator.d.ts +557 -0
  155. package/dist/resources/recordLockCoordinator.js +2597 -0
  156. package/dist/resources/recordLockCoordinator.js.map +1 -0
  157. package/dist/resources/replayLogs.js +5 -0
  158. package/dist/resources/replayLogs.js.map +1 -1
  159. package/dist/resources/replicatedApplyFailure.d.ts +16 -0
  160. package/dist/resources/replicatedApplyFailure.js +63 -0
  161. package/dist/resources/replicatedApplyFailure.js.map +1 -0
  162. package/dist/resources/scheduler/scheduler.js +3 -3
  163. package/dist/resources/scheduler/scheduler.js.map +1 -1
  164. package/dist/resources/search.d.ts +10 -4
  165. package/dist/resources/search.js +160 -40
  166. package/dist/resources/search.js.map +1 -1
  167. package/dist/resources/tracked.d.ts +5 -1
  168. package/dist/resources/tracked.js +74 -23
  169. package/dist/resources/tracked.js.map +1 -1
  170. package/dist/resources/transactionBroadcast.js +4 -6
  171. package/dist/resources/transactionBroadcast.js.map +1 -1
  172. package/dist/security/auth.js +59 -23
  173. package/dist/security/auth.js.map +1 -1
  174. package/dist/security/deferredAuthentication.d.ts +11 -0
  175. package/dist/security/deferredAuthentication.js +25 -3
  176. package/dist/security/deferredAuthentication.js.map +1 -1
  177. package/dist/security/jsLoader.js +9 -4
  178. package/dist/security/jsLoader.js.map +1 -1
  179. package/dist/server/REST.js +39 -5
  180. package/dist/server/REST.js.map +1 -1
  181. package/dist/server/http.d.ts +5 -1
  182. package/dist/server/http.js +34 -2
  183. package/dist/server/http.js.map +1 -1
  184. package/dist/server/mqtt.js +5 -1
  185. package/dist/server/mqtt.js.map +1 -1
  186. package/dist/server/serverHelpers/Headers.d.ts +2 -0
  187. package/dist/server/serverHelpers/Headers.js +6 -0
  188. package/dist/server/serverHelpers/Headers.js.map +1 -1
  189. package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
  190. package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
  191. package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
  192. package/dist/server/serverHelpers/Request.d.ts +5 -10
  193. package/dist/server/serverHelpers/Request.js +38 -136
  194. package/dist/server/serverHelpers/Request.js.map +1 -1
  195. package/dist/server/serverHelpers/contentTypes.d.ts +2 -0
  196. package/dist/server/serverHelpers/contentTypes.js +189 -15
  197. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  198. package/dist/server/serverHelpers/serverUtilities.d.ts +3 -3
  199. package/dist/server/serverHelpers/serverUtilities.js +110 -20
  200. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  201. package/dist/server/serverHelpers/uwsServer.js +4 -1
  202. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  203. package/dist/server/serverHelpers/webSocketCloseReason.d.ts +2 -0
  204. package/dist/server/serverHelpers/webSocketCloseReason.js +29 -0
  205. package/dist/server/serverHelpers/webSocketCloseReason.js.map +1 -0
  206. package/dist/server/storageReclamation.js +1 -1
  207. package/dist/server/storageReclamation.js.map +1 -1
  208. package/dist/server/threads/isolatedApplications.d.ts +47 -0
  209. package/dist/server/threads/isolatedApplications.js +171 -0
  210. package/dist/server/threads/isolatedApplications.js.map +1 -0
  211. package/dist/server/threads/logRotationTransport.d.ts +1 -0
  212. package/dist/server/threads/logRotationTransport.js +33 -0
  213. package/dist/server/threads/logRotationTransport.js.map +1 -0
  214. package/dist/server/threads/manageThreads.d.ts +64 -6
  215. package/dist/server/threads/manageThreads.js +261 -12
  216. package/dist/server/threads/manageThreads.js.map +1 -1
  217. package/dist/server/threads/socketRouter.d.ts +1 -0
  218. package/dist/server/threads/socketRouter.js +196 -13
  219. package/dist/server/threads/socketRouter.js.map +1 -1
  220. package/dist/server/threads/threadServer.js +30 -7
  221. package/dist/server/threads/threadServer.js.map +1 -1
  222. package/dist/utility/errors/hdbError.d.ts +24 -0
  223. package/dist/utility/errors/hdbError.js +58 -1
  224. package/dist/utility/errors/hdbError.js.map +1 -1
  225. package/dist/utility/hdbTerms.d.ts +2 -0
  226. package/dist/utility/hdbTerms.js +2 -0
  227. package/dist/utility/hdbTerms.js.map +1 -1
  228. package/dist/utility/logging/harper_logger.js +218 -38
  229. package/dist/utility/logging/harper_logger.js.map +1 -1
  230. package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
  231. package/dist/utility/logging/logGenerationCoordinator.js +184 -0
  232. package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
  233. package/dist/utility/logging/logRotation.d.ts +46 -0
  234. package/dist/utility/logging/logRotation.js +365 -0
  235. package/dist/utility/logging/logRotation.js.map +1 -0
  236. package/dist/utility/logging/logRotator.d.ts +1 -1
  237. package/dist/utility/logging/logRotator.js +172 -92
  238. package/dist/utility/logging/logRotator.js.map +1 -1
  239. package/dist/utility/npmUtilities.js +6 -4
  240. package/dist/utility/npmUtilities.js.map +1 -1
  241. package/dist/utility/watcherFallback.d.ts +0 -45
  242. package/dist/utility/watcherFallback.js +1 -125
  243. package/dist/utility/watcherFallback.js.map +1 -1
  244. package/dist/validation/configValidator.js +6 -3
  245. package/dist/validation/configValidator.js.map +1 -1
  246. package/index.ts +9 -0
  247. package/json/systemSchema.json +3 -0
  248. package/npm-shrinkwrap.json +177 -87
  249. package/package.json +13 -5
  250. package/resources/DESIGN.md +689 -19
  251. package/resources/DatabaseTransaction.ts +230 -17
  252. package/resources/LMDBTransaction.ts +21 -3
  253. package/resources/PrimaryRocksDatabase.ts +20 -7
  254. package/resources/RecordEncoder.ts +52 -10
  255. package/resources/Resource.ts +97 -13
  256. package/resources/ResourceInterface.ts +8 -0
  257. package/resources/RocksIndexStore.ts +2 -1
  258. package/resources/RocksTransactionLogStore.ts +111 -31
  259. package/resources/Table.ts +1273 -396
  260. package/resources/analytics/DESIGN.md +38 -0
  261. package/resources/analytics/write.ts +50 -17
  262. package/resources/auditStore.ts +460 -11
  263. package/resources/crdt.ts +22 -0
  264. package/resources/dataLoader.ts +3 -4
  265. package/resources/databases.ts +611 -147
  266. package/resources/derivedIndexRegistry.ts +56 -0
  267. package/resources/derivedIndexRuntime.ts +2292 -0
  268. package/resources/graphql.ts +3 -2
  269. package/resources/indexes/DESIGN.md +833 -0
  270. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +924 -46
  271. package/resources/indexes/fullTextDerivedIndex.ts +1165 -0
  272. package/resources/indexes/fullTextNativeBinding.ts +146 -0
  273. package/resources/indexes/hnswDerivedIndex.ts +531 -0
  274. package/resources/indexes/hnswPlaneBinding.ts +174 -0
  275. package/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts +181 -0
  276. package/resources/nodeIdMapping.ts +50 -0
  277. package/resources/record-locks.md +1407 -0
  278. package/resources/recordLock.ts +173 -7
  279. package/resources/recordLockCoordinator.ts +3082 -0
  280. package/resources/replayLogs.ts +5 -0
  281. package/resources/replicatedApplyFailure.ts +77 -0
  282. package/resources/scheduler/DESIGN.md +40 -0
  283. package/resources/scheduler/scheduler.ts +4 -4
  284. package/resources/search.ts +169 -49
  285. package/resources/tracked.ts +73 -22
  286. package/resources/transactionBroadcast.ts +4 -4
  287. package/security/DESIGN.md +175 -0
  288. package/security/auth.ts +53 -24
  289. package/security/deferredAuthentication.ts +24 -2
  290. package/security/jsLoader.ts +9 -4
  291. package/server/DESIGN.md +275 -0
  292. package/server/REST.ts +42 -6
  293. package/server/http.ts +34 -2
  294. package/server/mqtt.ts +6 -4
  295. package/server/serverHelpers/Headers.ts +5 -1
  296. package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
  297. package/server/serverHelpers/Request.ts +33 -131
  298. package/server/serverHelpers/contentTypes.ts +188 -15
  299. package/server/serverHelpers/serverUtilities.ts +157 -27
  300. package/server/serverHelpers/uwsServer.ts +4 -1
  301. package/server/serverHelpers/webSocketCloseReason.ts +25 -0
  302. package/server/storageReclamation.ts +2 -2
  303. package/server/threads/isolatedApplications.ts +157 -0
  304. package/server/threads/logRotationTransport.ts +40 -0
  305. package/server/threads/manageThreads.js +254 -12
  306. package/server/threads/socketRouter.ts +217 -11
  307. package/server/threads/threadServer.js +30 -7
  308. package/studio/web/assets/{Chat-BnCBegQz.js → Chat-DADFFGe_.js} +1 -1
  309. package/studio/web/assets/{FloatingChat-CoDW1ySS.js → FloatingChat-D_mI-rZ7.js} +3 -3
  310. package/studio/web/assets/{apiToken-Bwk5BLXW.js → apiToken-c2NiSDHa.js} +1 -1
  311. package/studio/web/assets/{applications-DHxGi7JH.js → applications-DktUqh7G.js} +1 -1
  312. package/studio/web/assets/{cssMode-s0cWI-_M.js → cssMode-Cs_75Xhw.js} +1 -1
  313. package/studio/web/assets/{editor-DNcRHK54.js → editor-19b-Y1IN.js} +1 -1
  314. package/studio/web/assets/{html-Bdssedlg.js → html-DiYEQMpB.js} +1 -1
  315. package/studio/web/assets/{htmlMode-CoDlJ3fw.js → htmlMode-CmR0y7P_.js} +1 -1
  316. package/studio/web/assets/{index-D6sxmFLR.js → index-Dm0rfkJ7.js} +5 -5
  317. package/studio/web/assets/{index.lazy-tmU5BS8s.js → index.lazy-7vqt2CC3.js} +1 -1
  318. package/studio/web/assets/{javascript-B8meVSTH.js → javascript-BWtCFuOt.js} +1 -1
  319. package/studio/web/assets/{jsonMode-DpIPd35T.js → jsonMode-Buzzbv9y.js} +1 -1
  320. package/studio/web/assets/{languageServices-C_5FMJzQ.js → languageServices-SqsFWfTM.js} +1 -1
  321. package/studio/web/assets/{lspLanguageFeatures-BIzNBzPK.js → lspLanguageFeatures-EMV5cmjo.js} +1 -1
  322. package/studio/web/assets/{notifications-CvZivSbh.js → notifications-CAB-LZWT.js} +1 -1
  323. package/studio/web/assets/{notifications-CQf18QKb.js → notifications-DRzmSRxM.js} +1 -1
  324. package/studio/web/assets/{profile-DdOwtntb.js → profile-BNKAl79n.js} +1 -1
  325. package/studio/web/assets/{regions-n69fwagr.js → regions-CUow_Zw2.js} +1 -1
  326. package/studio/web/assets/{register-PfWTCXWB.js → register-Dkt3WUMp.js} +2 -2
  327. package/studio/web/assets/{setComponentFile-Bg6O7X0S.js → setComponentFile-BZRfMD0N.js} +1 -1
  328. package/studio/web/assets/{setup-CUx_aUDl.js → setup-D_yiEPO2.js} +2 -2
  329. package/studio/web/assets/{status-D7BVKqX9.js → status-DhHh1Ge-.js} +1 -1
  330. package/studio/web/assets/{toggleHighContrast-DBSyXzMr.js → toggleHighContrast-D7L1PDtV.js} +1 -1
  331. package/studio/web/assets/{tsMode-BByKCjBS.js → tsMode-CCwLk1YS.js} +1 -1
  332. package/studio/web/assets/{typescript-DDLnLpw9.js → typescript-BP1j1mjn.js} +1 -1
  333. package/studio/web/assets/{useEntityRestURL-31CHGaHk.js → useEntityRestURL-D7bnYxLw.js} +1 -1
  334. package/studio/web/assets/{workers-pR3jRY9D.js → workers-tOuCNT17.js} +1 -1
  335. package/studio/web/assets/{xml-2iRnMhQO.js → xml-BSG_3mQT.js} +1 -1
  336. package/studio/web/assets/{yaml-Bf92gJpd.js → yaml-DxiLprBB.js} +1 -1
  337. package/studio/web/index.html +1 -1
  338. package/utility/DESIGN.md +55 -0
  339. package/utility/errors/hdbError.ts +54 -0
  340. package/utility/hdbTerms.ts +2 -0
  341. package/utility/logging/harper_logger.ts +210 -30
  342. package/utility/logging/logGenerationCoordinator.ts +196 -0
  343. package/utility/logging/logRotation.ts +367 -0
  344. package/utility/logging/logRotator.ts +196 -91
  345. package/utility/npmUtilities.ts +6 -4
  346. package/utility/watcherFallback.ts +0 -122
  347. package/validation/configValidator.ts +6 -3
@@ -0,0 +1,1407 @@
1
+ # Record locks — Design notes
2
+
3
+ `table.lock()`: the local (Phase 0) primitive and the cluster (Phase 1) ownership design built on it.
4
+
5
+ **Read this when:** touching `recordLock.ts`, `recordLockCoordinator.ts`, `DatabaseTransaction` lock handling, or anything user-facing about `lock()`.
6
+
7
+ Index of every design note: [DESIGN.md](../DESIGN.md).
8
+
9
+ ---
10
+
11
+ ## Record locks: the native key lock is the sole authority (`Table`/`DatabaseTransaction`/`recordLock`)
12
+
13
+ `table.lock(id, options?, context?)` (harper#483, Phase 0: one node, every worker thread) gives a caller exclusive
14
+ write access to one record. The sole authority is the rocksdb-js process-wide key lock — a shared in-memory
15
+ map keyed by `[Symbol.for('record-lock'), tableId, id]`. No write goes to the store or audit log for
16
+ `lock()` or `unlock()`. The record's version and stored bytes are unchanged when a lock is acquired or
17
+ released; only the native key is locked in memory. This design eliminates all durable lock state from
18
+ the on-disk format: durable LOCK/UNLOCK writes would produce version bumps that peers interpret as
19
+ out-of-order duplicates and discard.
20
+
21
+ **Phase 0 contract.** `lock()` is mutually exclusive only with other `lock()` calls on the same key.
22
+ Plain writes (`put`, `patch`, `delete`, `create`, `invalidate`, `relocate`) are never gated, parked,
23
+ or restaged — they proceed immediately at real wall-clock time. A holder write starts at the lock
24
+ acquisition time, so a later plain write wins under LWW unless the holder first wrote through an
25
+ unpinned mixed explicit transaction. That transaction's later timestamp becomes the handle's floor
26
+ so the holder cannot lose its own subsequent writes; ordering against plain writes between acquisition
27
+ and that transaction timestamp is best-effort. Use `lock()` when the caller needs to read-then-
28
+ conditionally-write without another holder interleaving, not to serialize arbitrary writers.
29
+
30
+ Consequences that shape the code:
31
+
32
+ - **`store.tryLock(lockKey, onUnlocked)` is the acquisition primitive.** It returns `true` immediately
33
+ if the key is free, or queues `onUnlocked` and returns `false`. `store.unlock(lockKey)` is ownerless —
34
+ any caller can release — and fires all queued callbacks. Because `unlock` is ownerless, the handle's
35
+ `released` flag (set atomically with `store.unlock` in the same thread as the lease timer) is what
36
+ prevents a stale holder from clearing a new holder's lock: once `released` is set, `release()` is a
37
+ no-op. The key is `lockAttemptKey(tableId, id)` = `[LOCK_KEY_PREFIX, tableId, ...id]`, distinct from
38
+ `getFromSource`'s bare-id single-flight lock.
39
+ - **`RecordLockHandle`** (`recordLock.ts`) carries `store`, `key`, `keyId`, `acquiredAt`, `expiresAt`,
40
+ `hold`, `released`, and `expired`. `release()` is synchronous: it sets `released`, clears the lease
41
+ timer, and calls `store.unlock(key)`. A lease timer sets `expired = true` then calls `store.unlock()`
42
+ on fire; any write staged through an expired or released handle throws 409 in
43
+ `DatabaseTransaction.save()` before the write reaches the store — from `lockNotHeldError()`, which
44
+ names the actual cause, since an expired lease and a handle already released (by `unlock()` or by
45
+ the commit) send a caller after different bugs. `acquireRecordKey` loops
46
+ `tryLock` → await wake → retry until acquired or `waitMs` elapsed (then 423). The contender wait
47
+ timer uses `.unref()` so it does not prevent process exit.
48
+ - **Re-entrancy is per-transaction.** `DatabaseTransaction.recordLocks` is a lazily allocated
49
+ `Map<store, Map<keyId, handle>>` (O(1) lookup). `registerRecordLock`, `recordLockFor`, and
50
+ `unregisterRecordLock` manage it. `lock()` consults it before calling `tryLock`; a re-entrant call
51
+ returns the existing live handle. A handle expired by its lease timer is pruned on next re-lock lookup
52
+ so a stale holder's write gets 409.
53
+ - **Release.** A transaction-scoped handle (the default) is in `link.recordLocks`; every commit or abort
54
+ calls `releaseRecordLocks()` which iterates and calls `handle.release()` on each non-hold handle.
55
+ `{ hold: true }` attaches the handle to the returned instance as `#lockHandle`; `unlock()` calls
56
+ `handle.release()` directly (synchronous, returns false if already released). `unlock()` also accepts
57
+ scoped handles: it calls `release()` early and unregisters the handle so the transaction commit does
58
+ not release it again. After any `unlock()` call `#lockWritable` is cleared so writes through the
59
+ instance are no longer accepted. When no iterators are open (`readTxnsUsed <= 1`) the read snapshot
60
+ is released and `snapshotFree` is set so subsequent reads see current state.
61
+ - **Staging model: scoped stages like `update()`; hold stays deferred.** `#reloadLocked` eagerly
62
+ calls `_writeUpdate(id, this.#changes, false)` for a fresh scoped acquisition, exactly as the
63
+ instance `update()` does, so a `TransactionWrite` exists on the transaction immediately and `save()`
64
+ is the ordinary `#savingOperation` path — there is no `#lockWritable` auto-restaging branch for
65
+ scoped. A second write on the same locked instance (after an earlier `save()` on it has already run)
66
+ needs its own `update()` call to create a fresh `TransactionWrite`, the same as any other resource.
67
+ Hold keeps the deferred model: `save()`'s `#lockWritable` branch (now gated on `this.#lockHandle.hold`)
68
+ calls `_writeUpdate` lazily at `save()` time, because the acquiring transaction may already have
69
+ committed before the holder ever writes. The expired/released-handle 409 lives in the write path —
70
+ `DatabaseTransaction.save()`'s guard on `operation.lockHandle`, plus the hold branch's own liveness
71
+ check in `Table.save()` — not duplicated for scoped, since its eagerly-staged write already carries
72
+ `lockHandle` into the same guard.
73
+ - **Read-your-writes in `#reloadLocked`.** Freshness always starts from the committed entry
74
+ (`primaryStore.getEntry(id)`, snapshot-free) so a hold lock sees concurrent committed writes rather
75
+ than a stale snapshot. A write earlier in the _same_ explicit transaction has not reached that
76
+ committed entry yet, so `#reloadLocked` looks up the tail `TransactionWrite` for the key
77
+ (`link.writesByKey`) and, if it (or an ancestor found via `priorStagedWrite`) has a `stagedEntry`,
78
+ takes the record from there — the same basis a chained write picks up (harper#1968): the record
79
+ comes from the prior staged write, the rest of the entry (version, audit chain, blob metadata) stays
80
+ the pre-transaction one.
81
+ - **Scoped lock in an explicit `transaction()` scope.** After acquisition, when no writes have been
82
+ staged yet (`link.writes.length === 0`), the transaction clock is pinned to `acquiredAt` so the
83
+ holder write wins over any pre-lock concurrent write. When prior staged writes already exist,
84
+ ordering is best-effort — no 409 is thrown. In an `ImmediateTransaction` context (no explicit
85
+ scope), a scoped lock's writes go through the same `update()`-style staging as above; each explicit
86
+ `update()`+`save()` cycle is stamped with `nextHolderVersion()` independently. A scoped lock acquired
87
+ outside any explicit `transaction()` scope persists until `unlock()` or the lease expires
88
+ (ImmediateTransaction's `releaseRecordLocks()` is a no-op for record locks).
89
+ - **Scoped → hold upgrade.** Calling `lock(id, { hold: true })` while the same transaction already
90
+ holds a scoped lock on the same key upgrades it via `handle.upgradeToHold(lease)`, which flips the
91
+ existing handle object to hold mode in place (new lease timer, `nextHolderVersion()` primed) rather
92
+ than retiring it and minting a new one — every instance already referencing the handle (same or a
93
+ different resource instance sharing the key) stays valid; retiring and replacing would invalidate
94
+ those other references (their `save()` would then throw 409 against a released handle). The upgrade
95
+ also detaches the scoped phase's eagerly-staged `TransactionWrite` (see the staging-model bullet
96
+ below) via `detachScopedUpgradeWrite`: hold staging is deferred and explicit-save-only, so a dangling
97
+ scoped write left in place would otherwise auto-commit at the transaction's sweep and clobber
98
+ whatever the hold write lands. The detached write is marked `.dropped` so a later explicit `save()`
99
+ on the instance that owns it falls through to the hold branch instead of resolving a dead reference.
100
+ The upgrade is gated on `!scoped.hold`; if the existing handle is already a hold the call is
101
+ re-entrant and returns the existing handle.
102
+ - **Concurrent `lock()` calls for one key on one link coalesce.** `Promise.all([T.lock(id), T.lock(id)])`
103
+ would otherwise have both calls reach `tryLock` before either registers, so the second parks against
104
+ the first. The first registers its in-flight acquisition (`registerPendingLock`); the second becomes a
105
+ follower that races that promise against its OWN timeout, then either takes the re-entrant path or
106
+ retries with whatever budget it has left. A follower that lands after its enclosing transaction has
107
+ closed must NOT retry: `lock()` re-resolves the context, which no longer points at that link, so the
108
+ handle it acquired would be registered on a fresh transaction that no commit or abort ever releases —
109
+ a leaked key lock until the lease expires. It throws 500 instead, matching the leader's own
110
+ post-acquisition guard.
111
+ - **The static entry point resolves the passed context before the ambient one.**
112
+ `Table.lock(id, options?, context?)` takes the same trailing context as the other static verbs,
113
+ normalized by `contextArgument()` exactly as `transactional()` does (a bare `DatabaseTransaction`
114
+ becomes the context slot holding it). Honoring it is load-bearing, not cosmetic: a caller with no
115
+ ambient context — a background job, a timer, a subscription callback — would otherwise land on a
116
+ bare `{}` whose ImmediateTransaction releases no record locks, so the native key would stay locked
117
+ for the whole lease and every other `lock()` on that record would fail 423 until it expired.
118
+ - **Timed-out waiter callback residue (rocksdb-js follow-up).** When `acquireRecordKey` times out and
119
+ throws 423, the `onUnlocked` callback registered via `tryLock(key, onUnlocked)` stays live in the
120
+ native map until the current holder eventually releases the key. rocksdb-js has no `deregisterCallback`
121
+ API, so there is no way to cancel it today. The leaked callback is harmless — it fires once, calls
122
+ `wakeResolve?.()` on an already-settled promise (no-op), and is then freed — but it is a small
123
+ unnecessary allocation per timed-out waiter. Track as a follow-up: rocksdb-js should expose a
124
+ cancelable wait-registration API so `acquireRecordKey` can deregister on timeout.
125
+ - **Crash / thread death.** A process crash releases all key locks (process-wide in-memory). A worker
126
+ thread termination releases its locks: rocksdb-js's `~DBHandle()` destructor calls
127
+ `lockReleaseByOwner(this)` on env teardown, releasing every key the terminated thread's handle held.
128
+ The lease timer is a soft bound in case the holder's event loop is blocked.
129
+ - **Not supported on LMDB.** `lock()` throws 501.
130
+ - **`lock()` is an in-process verb only.** `Resource.lock` is a static verb registered through
131
+ `transactional()` but no protocol reaches it: REST answers 501, `KNOWN_METHODS` does not include it,
132
+ and neither OpenAPI nor MCP enumerate it. Exposing lock/unlock over a protocol is a Phase 1 decision.
133
+ Acquisition itself has no authorization hook — lock() is not protocol-dispatched, so no
134
+ allowUpdate/allowCreate check runs when a caller acquires a lock.
135
+ - **`lock()` and `allowUpdate`:** writes through a held lock bypass per-table `allowUpdate`/`allowWrite`
136
+ hooks by the same trust model as any in-process `Table.update(id)` + set/save sequence.
137
+
138
+ Not in Phase 0, by design: replication of lock transitions, distributed grant, lease renewal,
139
+ subscription events for lock/unlock, and lock() on LMDB. Phase 1 adds the distributed grant — see
140
+ "Phase 1: cluster-wide `lock()` over replicated control entries" below. Lease renewal, subscription
141
+ events for lock/unlock, and lock() on LMDB remain out of scope.
142
+
143
+ **Acquisition timestamp and mixed transactions.** In an `ImmediateTransaction` context (no explicit
144
+ `transaction()` scope) every save — hold or scoped — is stamped by `handle.nextHolderVersion()` so
145
+ sequential saves each get a distinct, monotonically-increasing version. That stamp lives on the write
146
+ (`TransactionWrite.lockStamp`) and is
147
+ never assigned to the link clock: pinning `link.timestamp` would stamp every OTHER write staged on the
148
+ same context before the commit resets it — a concurrent write in the caller's own `Promise.all`, an
149
+ off-key write through the locked instance, the next operation in a retry or replay save loop — with
150
+ the lock's acquisition time, which LWW then silently drops against a newer record version. In an
151
+ explicit OPEN transaction, when the hold is the first write (no prior staged writes), the transaction
152
+ clock is pinned to `handle.acquiredAt`; subsequent saves reuse that pinned clock. When non-hold writes
153
+ were staged before the lock was acquired, the clock is left alone (best-effort ordering; no 409 is
154
+ thrown for the mixed-write case). After a lock-backed record change commits, its transaction timestamp
155
+ advances `handle.noteHolderVersion()`, so a surviving hold's later saves advance past that version
156
+ rather than going backwards and being dropped by LWW. A skipped or rolled-back change never advances
157
+ the floor. Consequently, a mixed transaction can
158
+ make later holder writes outrank a plain write whose timestamp falls between `acquiredAt` and the mixed
159
+ transaction timestamp. A caller-supplied future `context.timestamp` likewise remains the handle's floor
160
+ for the life of the lease; clamping it would put the next holder write behind the handle's own committed
161
+ version and recreate the silent-drop bug.
162
+
163
+ **Hold handles and re-entrancy scope.** A hold handle stays registered on the resource instance
164
+ (`#lockHandle`) and on the link until `unlock()` is called. Writing through the returned record after
165
+ the acquiring transaction committed is fine (each write auto-commits as an ImmediateTransaction).
166
+ Taking the lock again in a second `transaction()` scope issues a fresh `lock()` call rather than
167
+ relying on the first hold still being re-entrant in that scope.
168
+
169
+ **Untested scenarios (single-threaded unit tests).** One scenario cannot be exercised with a single
170
+ JS thread:
171
+
172
+ - _Abort during the `acquireRecordKey` await window._ The async gap between `tryLock` failing and the
173
+ `onUnlocked` callback is short in practice, and injecting an abort during that window requires two
174
+ concurrent threads.
175
+
176
+ ### Phase 1: cluster-wide `lock()` over amortized per-record ownership (`recordLockCoordinator`)
177
+
178
+ > **Phase 1 contract: exclusion-only.** A cluster `lock()` guarantees exclusive _admission_ of a
179
+ > critical section, and successor freshness after a clean handoff while the key's home still holds
180
+ > that handoff's dependency set. It adds no fencing generation and does not confirm locked writes to a
181
+ > quorum, so **it changes nothing about how two conflicting writes resolve** and it does not promise
182
+ > freshness once that dependency set is gone. Neither limitation is crash-only and both are reachable
183
+ > on a clean handoff.
184
+ >
185
+ > **§10 of the Phase 1 design below is the normative
186
+ > wording**: the routes into each limitation, why there is no caller-side mitigation, and the rule
187
+ > that none of it may be softened in the API docs. Read it before writing anything user-facing about
188
+ > `lock()`.
189
+ >
190
+ > One consequence bears on the code here rather than on the API: **the Phase 0 contract's relationship
191
+ > with ordinary writes is unchanged by Phase 1** — plain writes are still never gated and still
192
+ > resolve by last-write-wins, with no field added to them — which is the main thing the fenced arm
193
+ > would have given up. The fenced and quorum-confirmed alternatives are deferred, with their costs, to
194
+ > harper#2540.
195
+
196
+ Phase 1 keeps every Phase 0 mechanism and adds a cluster step on top of it. Nothing in core registers
197
+ a `ClusterLockTransport`, so in a core-only build the machinery is inert and `lock()` behaves exactly
198
+ as it did in Phase 0 — the whole feature is gated on harper-pro registering a transport.
199
+
200
+ **Three levels at three very different rates.** This is the shape the design note argues for, and the
201
+ reason the arbitration rule is one node rather than a quorum:
202
+
203
+ | level | rate | who owns it | what it costs |
204
+ | ---------- | --------------------- | ----------- | ------------------------------------- |
205
+ | home map | operator reconfigures | harper-pro | published out of band, never per lock |
206
+ | home node | derived, free | core | a hash over the map's `homes[]` |
207
+ | delegation | per key, bounded time | core | 1 RTT cold, **zero** while it is live |
208
+
209
+ `transport.homeMap(database)` hands core `(generation, homes[], homeIncarnation)`, where `homes[]`
210
+ names every node that participates in record locks — a home refuses a delegation to any node the map
211
+ does not name, so a node absent from it can neither arbitrate nor lock. The map is
212
+ **operator-agreed and immutable per generation**: an administrator publishes it, peers agree on its
213
+ digest before the feature is enabled, and nothing a node observes — an unreachable peer, a restart,
214
+ a partition — changes it. Core never computes topology, never advances a generation, and never
215
+ proceeds without a map: no map means no agreed ring, and a ring guessed from whoever looks reachable
216
+ is exactly the asymmetric-partition failure a single arbiter exists to remove. A key's **home** is
217
+ the rendezvous-hash winner over `homes[]` — chosen over a modulo because a generation change then
218
+ re-homes only the departing node's keys, and every re-homed key pays the recovery path on its next
219
+ lock.
220
+
221
+ **No consensus runs at any rate**, which is the point of the shape. An earlier revision agreed
222
+ `members[]` by single-decree consensus so an unreachable node could be rehomed automatically; that
223
+ was replaced on 2026-09-13 because the decision it automated — is this node briefly down, or gone? —
224
+ has a human authority who can simply state it. The price is stated rather than hidden: an
225
+ unavailable home's keys stay unavailable until an operator publishes a new generation. §4 of the
226
+ design note carries the reasoning and §9 records the rejected alternative.
227
+
228
+ A node that wants to lock `K` asks `K`'s home for a **delegation**: the exclusive right to admit
229
+ critical sections on `K` for a bounded time. While one is live, `lock()`/`unlock()` are pure Phase 0
230
+ — the local rocksdb key lock, no cluster message. **Releasing the application lock does not release
231
+ the delegation**, which is the whole amortization: a node writing the same record repeatedly pays one
232
+ round and then nothing, and the delegate is in practice the last writer. When another node wants the
233
+ key, its home recalls the delegation; the delegate stops admitting, drains what is in flight, and
234
+ writes the release.
235
+
236
+ **Two control entries, not four.** `LOCK_RELEASE = 12` and `LOCK_BARRIER = 13` are the lock action
237
+ nibbles in `auditStore.ts`. `LOCK_REQUEST = 9` and `LOCK_GRANT = 10` belonged to the Ricart–Agrawala
238
+ rule the design note replaces; that rule never shipped enabled, so those nibbles were **retired
239
+ rather than migrated** — and 9 has since been taken by eviction, which is why a migration was never
240
+ an option. Delegation request/grant/recall are unicast over the transport. The release stays on the
241
+ replicated log because it is what orders a handoff behind the delegate's own data writes; the
242
+ barrier (the recovery fence below) is on it because its log position is the whole point.
243
+
244
+ The release entry is written in its own transaction, with no primary-store write, and — unlike the `reload`
245
+ marker it is otherwise modeled on — **not** `LOCAL_ONLY`, because replicating it IS the send. Its
246
+ payload is `[key, requesterName, generation, homeIncarnation, counter]`, validated on exact tuple
247
+ length: a future version that grows it must bump the type rather than widen this one, since a
248
+ partially-understood release would clear a delegation on terms the sender did not intend.
249
+
250
+ **`recordId` is null and the key rides in the payload.** A control entry carrying the locked key as
251
+ its record id answers `_writeUpdate`'s keyed dedup lookup at exactly the holder's stamp, and
252
+ `RocksTransactionLogStore.getSync` returns it ahead of the record's own audit entry — silently
253
+ dropping the holder's first write. That is why the key is in the payload.
254
+
255
+ **Fencing tokens are ordered, not merely unique.** A delegation carries
256
+ `(generation, homeIncarnation, counter)`, compared lexicographically. `homeIncarnation` is durably
257
+ persisted and monotonic, supplied by harper-pro, and advanced once per **coordination incarnation** —
258
+ a process start or a coordinating-worker restart. Coordinator state including the delegation counter
259
+ is per-thread, so a replacement coordinating worker that kept the same incarnation would re-mint
260
+ tokens its predecessor issued. A random incarnation would make a stale reply
261
+ identifiable but not _orderable_: a home that restarts and re-issues counter 1 after having issued
262
+ counter 50 would let a delayed counter-50 reply defeat its successor.
263
+
264
+ **The home always outwaits its delegate.** A grant's deadline on the home is the delegate's lease plus
265
+ `LOCK_LEASE_SKEW_MS`, and both sides measure on their own monotonic clock — no remote timestamp is
266
+ ever compared against a local one. So the delegate stops admitting first, and the home cannot re-grant
267
+ a key its previous delegate still believes it holds. The eviction rule is deliberately **asymmetric**,
268
+ and that asymmetry is the safety argument: a delegate may drop a delegation early, a **home may never
269
+ forget one before its expiry**.
270
+
271
+ **A coordinator never grants over live authority it cannot see.** Two different mechanisms, because
272
+ the two cases are different:
273
+
274
+ - **In-process transport swap** (a component reload re-registering a transport). The successor
275
+ **adopts** the predecessor's delegations and grants in its constructor, before the predecessor is
276
+ closed. The transport object changed; this node's delegations and the handles they admitted did not.
277
+ - **Cold start** (nothing to adopt: a process restart, or a worker taking coordination over from one
278
+ that died). A previous incarnation may have delegations still admitting, and it left no record.
279
+ Nothing external bounds them — the map is immutable, so its generation does not advance merely
280
+ because a process or a worker restarted — so **core enforces this one itself**:
281
+ `#grantableAfterMono` refuses to grant as a home until `DELEGATION_LEASE_MS + skew` after **that
282
+ coordinator was constructed**. Not process start: `performance.now()` and `timeOrigin` are
283
+ process-wide inside a worker too, so a process-anchored horizon reads as long elapsed in a
284
+ replacement coordinating worker — and not thread start either, since a thread can take coordination
285
+ ownership long after it booted. It costs availability on this node's own share of the ring and
286
+ nothing elsewhere; an adopted successor inherits the predecessor's horizon rather than starting a
287
+ new one, so a transport reload is free. A deployment that can prove a previous incarnation issued
288
+ nothing overrides it through `ClusterLockTransport.grantableAfterMono`.
289
+
290
+ `Table.lockCoordinator` also does **not** close the coordinator when the transport merely goes away:
291
+ harper-pro unregisters without a standalone claim during a reconnect, and closing there would discard
292
+ this node's record of what it has granted. Only an explicit standalone claim clears it.
293
+
294
+ **A delegation is authority within one generation.** `#liveDelegation` compares the delegation
295
+ token's generation against the current one and drops it on a mismatch, because a generation change
296
+ may have re-homed the key to a node that knows nothing of this token. Bounding the window between a
297
+ delegate noticing a new generation and the new home granting is the design note's §4.3 drain —
298
+ operator-sequenced, and harper-pro's to hold, not core's.
299
+
300
+ **Recall revokes capability, not just admission.** Waiting for live admissions is not enough on its
301
+ own: a caller that staged a write and then called `unlock()` leaves nothing for a drain to wait on,
302
+ but its write is still uncommitted and would land after the successor was admitted. So a delegation
303
+ retains a revoker for every handle it admitted (`registerAdmission`), and surrender, expiry and
304
+ `close()` all call them. The admission carries its delegation's **token**, and `release()` and
305
+ `registerAdmission()` both take it back: a key's delegation can be replaced while a handle is still
306
+ open, and an untokened release from a superseded handle would decrement the successor's admission
307
+ count and let it be surrendered while its own callers were still inside — `handle.revokeLease()` expires the handle ahead of its lease, and the
308
+ commit-time fence in `DatabaseTransaction` then rejects the staged write immediately before the native
309
+ commit submits. That fence runs only when the transaction actually holds a lease-protected write
310
+ (`hasLeaseProtectedWrite`, reset by `clearWrites()`), so a bulk transaction of plain writes in a
311
+ core-only deployment pays nothing for it.
312
+
313
+ **A delegation outlives any one lock, and that is the amortization.** `DELEGATION_LEASE_MS` is
314
+ deliberately longer than the longest lock lease it will admit. A delegation sized to the caller's own
315
+ lease has no room left for the next lock, so every repeat `lock()` would renew and pay a round trip —
316
+ which is precisely the cost this design exists to remove. A delegate's deadline is also anchored at
317
+ the moment it SENT the request, not at the moment the reply arrived, so a delayed reply cannot give it
318
+ more time than the home is holding the key for; a reply that outlived its own delegation is discarded
319
+ rather than installed.
320
+
321
+ **`ts_R` is chosen before the write and lives only in the payload.** The writer takes the store's
322
+ monotonic timestamp for the holder's stamp and lets the control entry commit at its own fresh time, so
323
+ an entry can never land behind a peer's replication cursor.
324
+
325
+ **An entry's identity is bound to the node that wrote it.** `applyEntry` takes the author from the
326
+ audit header, never from the payload, and ignores a release whose payload names anyone else. Without
327
+ that, a peer could write a release naming another node and clear a delegation it does not hold. A
328
+ release also carries the **whole fencing token**, not just the counter, and is matched against the live
329
+ grant on all three components: a home that restarts begins counting again, so a counter-only match
330
+ would let a delayed release from a previous incarnation clear a live grant while its delegate is
331
+ still admitting.
332
+
333
+ **Bounded state.** Delegations are capped per database and per requester, expiry work is bounded per
334
+ tick, and clean-handoff dependency sets have a separate, larger LRU cap. A fixed add-only Bloom
335
+ filter distinguishes a truly virgin key from an evicted dependency set while this coordinator has
336
+ observed the whole generation; after a cold start, ownership gap, or generation change, every
337
+ unremembered key conservatively takes recovery.
338
+
339
+ **Membership is fail-closed.** No agreed home map, no named homes, an unreachable home, a closed
340
+ coordinator, or a call on a thread that does not own coordination all reject with a retryable 503
341
+ rather than downgrading to a node-local lock — which would hand two nodes one key. An unreachable node
342
+ blocks only the keys it homes, which is the availability property the whole redesign exists for.
343
+
344
+ **An exhausted wait is classified from the last reply a home completed, never from a `timeout`.**
345
+ `#requestRemotely` races the home against the caller's remaining budget and
346
+ synthesizes `reason: 'timeout'` when the budget wins; that reply is this node's own deadline, not
347
+ something the home said, so it is evidence-free. `acquire()` keeps the last completed reply in
348
+ `lastCompleted`, bound to the `(home, generation)` that produced it; at the terminal branch it
349
+ re-reads the map and accepts an observation — the carried one and the pass's own reply alike — only
350
+ under the generation that is current then, since one can be activated while the last probe is in
351
+ flight. It classifies from what survives: a wait that watched the key held answers 423 even though its final probe was
352
+ cut short, while a later `not-home` — the fresher fact — still answers 503. It also stops before a
353
+ backoff that would reach the deadline, since the probe after it could only come back as that same
354
+ synthetic timeout. Do not restore "the last reply wins": the retry loop's final pass is by
355
+ construction the one most likely to be truncated, so that rule reports a contended key as a
356
+ coordination failure at random.
357
+
358
+ **`{ scope: 'node' }`** opts out of the cluster step and keeps exact Phase 0 semantics, which by
359
+ design permits simultaneous holders on different nodes. An **explicit** `{ scope: 'cluster' }` with no
360
+ transport rejects 503 rather than silently returning the weaker lock, and a transaction that already
361
+ holds a key node-scoped cannot take a cluster lock on it (409) — including through the concurrent-lock
362
+ coalescing path, where a follower would otherwise inherit the leader's weaker handle.
363
+
364
+ **Routing and exclusion from record surfaces.** The replicated-event consumer in `Table.ts` dispatches
365
+ the release entry to the table's `LockCoordinator` before it resolves a resource, so it never reaches
366
+ `_writeUpdate`, and `stageWrite` keeps it off the per-key write chain. Every surface that reports
367
+ audit entries as record activity filters it through `isLockControlType`: the subscriber listener
368
+ (ahead of the `rawEvents` branch, which otherwise forwards every type verbatim), the
369
+ `subscribe({ startTime })` replay, the `previousCount` backfill, and `getHistory()`.
370
+ `getHistoryOfRecord()` excludes it already by matching on record id.
371
+
372
+ **Rolling upgrade.** Unlike the reload marker this entry is modeled on, the lock nibble is not
373
+ `LOCAL_ONLY`, so a peer that does not understand it must not be sent it. harper-pro gates the send
374
+ path on a **versioned** `recordLocks` capability: the versions are mutually exclusive and a node
375
+ advertises exactly one, because a cluster running both arbitration rules would have two independent
376
+ arbiters for one key.
377
+
378
+ **Audit-log surface.** The release entry appears in `read_audit_log` with a null record id and type
379
+ `lockRelease`. That is deliberate — it is protocol traffic on the table's own log — and it is what
380
+ makes control-entry volume per lock a thing the measurement gate (harper-pro#824) has to report.
381
+
382
+ **Dual-clock (harper#2412 / rocksdb-js#811).** Phase 1 keeps Phase 0's stamping — the pinned
383
+ acquisition clock, the handle's version floor, and the mixed-transaction rules — unchanged. Nothing in
384
+ the delegation path assigns a record version.
385
+
386
+ **Successor freshness.** A clean release carries the delegation's inherited `(origin → position)`
387
+ set. The home merges the trusted release origin at the release entry's own log position, retains the
388
+ result after clearing the grant, and sends it with the successor grant. The successor remains pending
389
+ and recallable until `ClusterLockTransport.establishLockFreshness()` has made every dependency applied
390
+ and visible. Missing lineage selects that transport's weaker reachable-member recovery barrier;
391
+ failure or timeout returns 503 and hands the grant back without discarding retained lineage. The
392
+ cached-delegation branch does none of this work. Harper-pro's operator-agreed home map and transport
393
+ implementation remain the enablement boundary (harper-pro#825 / companion work on #822).
394
+
395
+ **Recovery fence.** The recovery barrier's position is a `lockBarrier` control entry (nibble 13,
396
+ payload `[1, nonce]`, harper#2625): a replicated no-op the probed member commits after the probe, so
397
+ it is appended after every transaction that member had committed — the one ordering no log-key head,
398
+ received tail, or sender-emitted marker gives, since entries are appended in commit order rather than
399
+ key order. `writeLockBarrier(database, table, nonce)` in `recordLockCoordinator.ts` writes one — strictly
400
+ this node's own commit through `Table.writeLockControlEntry`, never the transport's `writeControl`
401
+ hook, since the caller is the transport and the fence must be a position in this origin's log — and
402
+ resolves to the entry's log position; the transport supplies the nonce it will match the entry on.
403
+ The coordinator ignores the entry on receipt, and every `isLockControlType` exclusion above covers it. `establishLockFreshness()` receives the wait remaining
404
+ on the lock deadline. The harper-pro operation and drain are harper-pro#822's; a recovery marker with
405
+ no barrier fails closed.
406
+
407
+ ## Phase 1 design: amortized per-record ownership
408
+
409
+ Design note for harper#2498 / harper-pro#822 (Phase 1 of harper#483). It replaces the
410
+ Ricart–Agrawala arbitration rule the harper#2498 branch originally implemented. Phase 0 (harper#2462) is
411
+ unchanged and remains the local primitive everything here builds on.
412
+
413
+ Planning review cleared this design over ten rounds; the rounds that changed it are recorded in the PRs harper#2498, harper#2613, harper#2627 and harper#2667.
414
+
415
+ ### 1. The problem with what is on the branch
416
+
417
+ `table.lock(id)` on the branch runs Ricart–Agrawala: the requester writes `LOCK_REQUEST` and waits
418
+ for a `LOCK_GRANT` from **every** participant (`resources/recordLockCoordinator.ts:619`). Core sets
419
+ `agreedDown` for nobody, so nothing is ever excluded from that set, and harper-pro#822 states the
420
+ consequence in its own limitations list: a crashed peer blocks new cluster locks for its databases
421
+ until it leaves `hdb_nodes`. Unanimity is not a high-availability protocol.
422
+
423
+ Two costs follow from the same rule. Every acquisition is `P+1` durable commits and up to `P²−1`
424
+ frame deliveries for `P` participants (each participant's grant is a transaction-log entry
425
+ replicated to every subscriber), and every acquisition pays a full network round even when the same
426
+ node locked the same key a millisecond ago. `P=12` — a size we run today — is 13 commits and 143
427
+ deliveries **per lock**.
428
+
429
+ ### 2. The invariants
430
+
431
+ **Only the first of these two ships in Phase 1**, and the second is narrowed rather than met. They
432
+ are stated together because they are what the design is _aimed_ at and what the deferred arms would
433
+ close; §10 is the normative wording for what `lock()` actually promises, and it is the text to read
434
+ before writing anything user-facing.
435
+
436
+ > **Exclusive admission.** At most one node may admit a critical section for
437
+ > `(database, table, key)` at any instant. Successor authority must exclude every predecessor
438
+ > _capability_ that can still admit or commit — not merely every predecessor handle.
439
+ >
440
+ > **Successor freshness.** A node admitted after a predecessor has applied the predecessor's
441
+ > committed writes to that key before it admits.
442
+
443
+ Freshness holds unconditionally on the clean-handoff path (§7.1). It is **explicitly narrowed on the
444
+ recovery path** (§7.2): a barrier over reachable members cannot observe a write the crashed
445
+ predecessor committed but never replicated, so the invariant as written is a promise this design
446
+ cannot keep in recovery mode for free.
447
+
448
+ **That narrowing is the shipping guarantee.** The §10 decision is **exclusion-only** (§7.3 (b)):
449
+ `lock()` promises exclusive _admission_, and successor freshness after a clean handoff _while the
450
+ home still holds that handoff's dependency set_ — a home restart, home-map change or cap eviction routes
451
+ the successor to the recovery barrier even after a clean release. It promises
452
+ neither conflict-ordering fences nor quorum-confirmed settlement, so the exclusion invariant above
453
+ ships narrower than it is written on two counts — the commit half holds only up to the
454
+ pre-submission expiry fence, and a predecessor's write can still outrank its successor's under LWW.
455
+ §10 states exactly what is and is not promised and is the normative wording; the two arms that would
456
+ close the rest are deferred to harper#2540, not rejected. Where the sections below describe fenced or
457
+ quorum-confirmed mode, they are describing that issue's scope, not this one's.
458
+
459
+ They are separate, and expiry establishes neither on its own. Phase 0 enforces part of the first:
460
+ expiry is checked synchronously on every staged write and again immediately before the native commit
461
+ submits (`resources/DatabaseTransaction.ts:1213`). That fence is reused unchanged — only the thing
462
+ that issues the lease changes — but it neither settles an already-submitted commit nor makes an
463
+ asynchronous replica fresh, which is what §6 and §7 are for.
464
+
465
+ ### 3. Structure
466
+
467
+ Three levels, at three very different rates.
468
+
469
+ **Level 1 — the home map (rare, operator-agreed, immutable).** Per database, a map
470
+ `(generation, homes[])`, published by the operator through harper-pro and never derived from
471
+ liveness. It is immutable for the life of its generation: nothing a node observes — an unreachable
472
+ peer, a restart, a partition — changes it. Consensus does not appear at this level at all; agreement
473
+ is that every node holds the same generation, checked by digest before the feature is enabled (§4).
474
+ A node that cannot obtain the current map stops acting.
475
+
476
+ **Level 2 — the home node (derived, free).** Within a generation the arbiter for a key is a
477
+ rendezvous hash over `homes[]` (§4.4). A single arbiter per key is trivially exclusive, which removes
478
+ the entire grant state machine — no deferral queues, no `(tsR, nodeName)` tiebreak, no synthesized
479
+ grants, no split votes, no `INQUIRE`/revocation protocol.
480
+
481
+ **Level 3 — the delegation (the amortization).** A node that wants to lock `K` asks `K`'s home node
482
+ for a _delegation_: the exclusive right to admit critical sections on `K`, for a bounded time. With
483
+ a live delegation, `lock()`/`unlock()` are pure Phase 0 — the local rocksdb key lock, no cluster
484
+ message. Releasing the application lock does **not** release the delegation. So a node writing the
485
+ same record repeatedly pays one round, then nothing, and the delegate is in practice the last writer.
486
+
487
+ ```
488
+ first lock on K from node B steady state (B keeps locking K) C wants K
489
+ B → home(K): DELEGATE(K) B: local key lock only C → home(K): DELEGATE(K)
490
+ home → B: GRANTED(K, tok, until) (zero cluster messages) home → B: RECALL(K, tok)
491
+ B: local key lock, run B: drain (§6), RELEASE entry
492
+ home → C: GRANTED(K, tok+1, …)
493
+ ```
494
+
495
+ Delegations are **volatile**; the home map is **durable, and written by an operator**. That split is
496
+ the design: the only durable, agreed state changes at the rate an administrator reshapes the cluster,
497
+ never on an acquisition and never on an ordinary write.
498
+
499
+ ### 4. The home map
500
+
501
+ harper-pro owns this, because it owns topology. The map is **operator-agreed and immutable per
502
+ generation**: an administrator publishes it through harper-pro, and no node ever derives, proposes or
503
+ advances one from what it observes.
504
+
505
+ #### 4.1 What core consumes
506
+
507
+ Per database, `(generation, homes[])`:
508
+
509
+ - `generation` is a monotonic number. It is the high-order component of the fencing token (§5.1), so
510
+ it must never go backwards, and a delegation minted under one generation is not honoured under
511
+ another.
512
+ - `homes[]` names **every node that participates in cluster record locks** for the database — not
513
+ only the ones an operator thinks of as arbiters. It is one set and not two because a home refuses a
514
+ delegation to any node the map does not name (§5), which is what keeps a decommissioned node from
515
+ taking one: a node absent from `homes[]` can therefore neither home a key nor lock one. Rendezvous
516
+ hashing then makes each listed node the arbiter for its own share of the ring, so the second list
517
+ would buy nothing. It is still a **lock-participation map, not a residency directive** — it says
518
+ who may arbitrate and lock, not where records live — and it is not derived from `hdb_nodes`, which
519
+ is LWW-replicated and therefore not agreed.
520
+
521
+ Core fails closed when no map is available and never guesses a ring. `homeIncarnation` (§5.1) rides
522
+ alongside as the one remaining durable per-node datum: a monotonic counter persisted by harper-pro
523
+ with the node's own identity, and advanced once per **coordination incarnation** — §5.1 states the
524
+ rule, and it is not once per process.
525
+
526
+ **Agreement is a digest comparison, not a protocol.** Before the feature is enabled for a database,
527
+ peers exchange a digest of `(generation, homes[])` and refuse to participate on a mismatch. There are
528
+ no ballots, no acceptors, no promises and no configuration certificates, because nothing is being
529
+ decided at run time — the decision was made by the operator and published.
530
+
531
+ **A digest check is not a freshness check**, and §4.3 is where that is paid for. It proves two nodes
532
+ that can talk to each other hold the same map; it cannot tell a node that its own map is a generation
533
+ behind, because a stale map is internally consistent. Nothing here bounds a node that never receives
534
+ a new generation — only §4.3's staged transition, with acknowledgements bound to incarnation and one
535
+ durable activation record, does.
536
+
537
+ #### 4.2 Why the operator, and what that buys
538
+
539
+ The hard question in an ownership map is not _how_ to change it but _whether to_: an unreachable
540
+ node may be briefly down or permanently gone, and those want opposite answers. A protocol that
541
+ decides for itself has to infer intent from timeouts, and the price of inferring it safely is a
542
+ durable consensus subsystem. Round 2's counterexample is the proof, and is kept here so the cheap
543
+ version is not reinvented:
544
+
545
+ > Generation `g` = `{A,B,C}`. `A,B` accept successor configuration `X = {A,D,E}`; `X` installs and
546
+ > stays live on `D`/`E` renewals. `B` restarts and forgets its acceptance; `C` never learned `X`; `A`
547
+ > is unreachable from `B`/`C`. After any quarantine, `B` and `C` observe generation `g` expired and
548
+ > accept a different successor `Y = {B,C,F}`. `X` and `Y` now renew on disjoint majorities and derive
549
+ > different homes for the same key.
550
+
551
+ Waiting out old leases cannot extinguish a configuration that has already been chosen and renews on
552
+ its _new_ membership, and there is no timing argument that substitutes for persisted promises. An
553
+ operator-published map does not have this shape at all: there is exactly one authority, it is
554
+ external, and a node either holds the published generation or refuses to act.
555
+
556
+ So the intent that a protocol would have had to infer is simply **stated**:
557
+
558
+ | the operator means | what they do | what the cluster does |
559
+ | ------------------------------------------ | ------------------------------------------------- | --------------------------------------------------------------------- |
560
+ | this node is temporarily down | nothing | its keys fail closed; every other home keeps serving |
561
+ | this node is permanently gone, or replaced | stop it, then publish generation `g+1` without it | §4.3's staged transition, then its keys are served by their new homes |
562
+
563
+ The cost of that is stated just as plainly, and it is the reason this is a decision rather than an
564
+ optimization: **an unavailable home's keys stay unavailable until an operator acts.** Automatic,
565
+ failure-driven rehoming is not deferred work — it is the thing being declined, and it returns only if
566
+ §10's measurements show the manual path is operationally unacceptable.
567
+
568
+ #### 4.3 Changing the map, and restarts
569
+
570
+ A generation change is **fail-closed and operator-sequenced**, because the danger in any ownership
571
+ change is a key acquiring a second arbiter while a delegation issued by the first is still live.
572
+ Delegations are volatile and bounded, so the transition is a drain rather than an agreement — but
573
+ publishing `g+1` is not by itself what stops `g`, and the note must not pretend otherwise:
574
+
575
+ 1. **Stage** `g+1` to every node named in `homes(g)` and in `homes(g+1)`.
576
+ 2. **Quiesce.** Each of those nodes acknowledges that it has stopped granting and stopped honouring
577
+ delegations under `g`, and **its acknowledgement carries its `homeIncarnation`** (§5.1). A node
578
+ that does not acknowledge must be **externally fenced** — stopped, isolated, or powered off —
579
+ before the transition may continue. This is the step that carries the whole safety argument, and
580
+ it is why the operator, not a protocol, is the authority: **declaring a node removed is not a
581
+ fence; stopping it is.**
582
+ 3. **Drain.** Wait `maxDelegationMs + skew` after the last acknowledgement or fence. That interval
583
+ bounds every delegation `g` could have issued before it stopped.
584
+ 4. **Activate.** One **durable activation record**, issued by the control plane, carries the staged
585
+ generation, the routing-capability version (§4.4), every acknowledgement with the incarnation it
586
+ was given under, the external-fence attestations, and a not-before instant. A node answers
587
+ `homeMap()` with `g+1` only once that record is active. Grants then resume under `g+1`; keys whose
588
+ home did not move are affected by the drain exactly as much as keys that did, because an unchanged
589
+ home is still a new arbiter under a new token prefix.
590
+
591
+ **An acknowledgement dies with the incarnation that gave it.** Round 10's counterexample, and the
592
+ reason step 2 records an incarnation rather than a node name: `A` acknowledges quiescence, then
593
+ restarts during the drain. Its new incarnation has none of that state and still holds `g` locally.
594
+ If the operator activates `g+1` on `A`'s old acknowledgement, `A` waits out its own §4.3 restart
595
+ quarantine and then serves `g` indefinitely, alongside whoever `g+1` homes those keys on. Binding the
596
+ acknowledgement to `homeIncarnation` invalidates it the moment `A` restarts, so activation blocks
597
+ until `A` re-acknowledges in its new incarnation or is fenced.
598
+
599
+ The record is **one-shot per reconfiguration, not a renewal**. Steady-state locking never contacts
600
+ the control plane; only a transition does. That is the whole distinction from the continuously leased
601
+ generation capabilities §9 rejects, and it is why this is a strengthening of the chosen design rather
602
+ than a move toward that one.
603
+
604
+ **Why step 2 is not optional.** Without it, a node that never receives `g+1` keeps serving `g` from
605
+ its own retained copy, and nothing in a one-time digest check can tell it that its internally
606
+ consistent map is stale. `A` and `B` hold `g={A,B}`; `A` partitions; the operator declares `A`
607
+ removed and publishes `g+1={B}`; `B` drains and re-homes `A`'s keys; `A` — still running, still
608
+ consistent with itself — grants one of them to whoever it can still reach. Two holders under one key.
609
+ A drain measured from _publication_ does not bound that, because `A` never stopped.
610
+
611
+ **The generation is also monotonic, and enforced as such.** A node refuses a map whose generation is
612
+ below one it has already acted under, per database — the scope the generation itself has. Core's half
613
+ lasts as long as the coordinating thread; remembering it across a restart, and across threads, is
614
+ harper-pro's. The rollback route here is a configuration
615
+ restore or a partial publish rather than a protocol bug, which is exactly why it is refused rather
616
+ than assumed away: a generation is the high-order component of every fencing token (§5.1), so
617
+ re-minting under an older one issues tokens ordering _below_ ones already handed out. Core enforces
618
+ this for the life of its process; remembering it across a restart is harper-pro's half.
619
+
620
+ **A restarted home is the one interval core enforces itself.** Delegations are volatile, so a cold
621
+ coordinator has no record of what a previous incarnation granted, and the generation does not change
622
+ on a restart — there is no external event to hang the interval on. Core therefore refuses to grant as
623
+ a home until `DELEGATION_LEASE_MS + skew` has elapsed **since that coordinator was constructed**, on
624
+ its own monotonic clock. It need not remember what it granted, only that everything it could have
625
+ granted has expired.
626
+
627
+ **Construction, not process start and not thread start**, because neither of those is sound. A
628
+ worker's `performance.now()` and `timeOrigin` are process-wide, so a process-anchored horizon reads
629
+ as long elapsed in a replacement coordinating worker; and a thread can take coordination ownership
630
+ long after it booted, so a thread-anchored one has the same hole. Construction is the earliest
631
+ instant core can prove nothing else was granting under. Where a predecessor's authority _is_ known —
632
+ an adopted transport swap, or an unregister and re-register inside one thread — the horizon is
633
+ waived or carried rather than recomputed.
634
+
635
+ That quarantine costs availability, and the cost is real: for keys a node homes, the first cluster
636
+ lock after a cold start waits out the interval. It does not affect keys homed elsewhere — the node
637
+ acquires those immediately — and it is bounded by the delegation lease, not by an operator's response
638
+ time. A deployment that can prove a previous incarnation issued nothing (a fresh database, a first
639
+ start) may override it through the transport, which is the only party that knows that.
640
+
641
+ #### 4.4 Routing — exactly one algorithm
642
+
643
+ `home(K) = argmax_{m ∈ homes} H(nodeName(m) ‖ 0x00 ‖ canonicalKey(K))`, ties broken by the lexically
644
+ greater `nodeName`. Rendezvous (highest-random-weight), **not** modulo indexing: a generation change
645
+ then moves only the keys homed to a departing node.
646
+
647
+ `canonicalKey(K) = utf8(database) ‖ 0x00 ‖ utf8(table) ‖ 0x00 ‖ orderedBinary(key)`, using the same
648
+ ordered-binary key encoding the primary store already uses, so two nodes cannot disagree about the
649
+ bytes.
650
+
651
+ **`H` and `canonicalKey` are versioned by the protocol capability, not by the generation.** A
652
+ generation is operator-published and says nothing about which hash a binary implements, so two
653
+ binaries can accept the same map and still derive different homes for a key — two arbiters, no
654
+ disagreement anyone can observe. The routing version therefore rides on the mutually-exclusive
655
+ `recordLocks` capability that already gates this protocol (§11), so nodes that would hash differently
656
+ cannot both be enabled in one cluster.
657
+
658
+ ### 5. Exclusion
659
+
660
+ - **One arbiter.** Within a generation every node derives the same home from the same `homes[]`, and
661
+ a digest mismatch stops a node acting rather than letting it derive a second ring (§4.1).
662
+ - **Across a generation change** — §4.3's drain.
663
+ - **Across a home restart** — §4.3's quarantine, which core enforces on its own process clock. The
664
+ timing argument is sufficient here because a delegation's liveness never migrates to a new set of
665
+ holders: it is one node, one bounded lease.
666
+ - **A delegate that loses contact** stops admitting when its delegation expires, enforced by the
667
+ existing commit-time fence. The home may re-grant only after its own issue instant plus
668
+ `durationMs + skew`, on the home's own monotonic clock.
669
+
670
+ #### 5.1 Fencing tokens are ordered, not just unique
671
+
672
+ A delegation's fencing token is `(generation, homeIncarnation, delegationCounter)`, compared
673
+ lexicographically. `homeIncarnation` is a **durably persisted monotonic counter**, carried on the home
674
+ map (§4.1) and incremented once per **coordination incarnation** — a process start _or_ a
675
+ coordinating-worker restart — not a random id, and not once per process. Coordinator state, the
676
+ delegation counter included, is per-thread: a replacement coordinating worker starts counting from
677
+ zero, so an incarnation that did not advance with it would let the new worker re-mint tokens its
678
+ predecessor already issued, and would leave §4.3's incarnation-bound quiescence acknowledgements
679
+ attesting to state that restart discarded. A random incarnation makes a
680
+ stale reply identifiable but not orderable, and a home that restarts and re-issues counter 1 after
681
+ having issued counter 50 would let a delayed counter-50 write defeat its successor. The token must
682
+ also survive key deletion and re-creation, which it does because it is scoped to the home and the
683
+ generation, not to the record.
684
+
685
+ #### 5.2 Lease transfer and local handle bounds
686
+
687
+ A home grants ten seconds and the reply arrives eleven seconds later; a receiver that starts its
688
+ clock at receipt overlaps the next delegate.
689
+
690
+ - The requester reads its monotonic clock at **send** (`t_send`) and sets its delegation deadline to
691
+ `t_send + durationMs`. Never from receipt, never from `deadline − Date.now()`.
692
+ - A reply arriving after `t_send + durationMs − minUsefulMs` is **rejected**, not clamped: a
693
+ delegation with no usable window is a retry, not a hold.
694
+ - Every message carries `(generation, homeIncarnation, delegationCounter, requestId)` and is bound to
695
+ the authenticated replication origin. A reply that does not match the requester's current generation
696
+ and its outstanding request is discarded; a validly authenticated node that the current generation
697
+ does not name can neither obtain a delegation nor clear one. **The `requestId` half is not
698
+ implemented** — the wire carries no request identity, correlation is the transport's per-call
699
+ promise, and the home therefore cannot tell a duplicate or delayed request from a genuine renewal.
700
+ Core closes the resulting two-holder path by refusing to hand back a grant while any delegation for
701
+ the key is held; the residue, and the protocol fix, are harper#2582. The generation half **is**
702
+ implemented: the map is re-read after the round and a grant minted under a superseded generation is
703
+ handed back rather than installed.
704
+ - **Every local handle is bounded by `min(requested lease, delegation deadline)`** — including
705
+ re-entrant acquisition and the `{hold: true}` upgrade. The branch's
706
+ `upgradeToHold` already clamps to the granted round's deadline rather than extending it
707
+ (`resources/recordLock.ts:305`); that clamp is retargeted, and must not be dropped along with the
708
+ round it currently reads.
709
+ - **Clock assumptions, stated rather than implied:** every deadline is compared only against readings
710
+ of the _same_ node's `performance.now()`; no remote instant is ever compared against a local one,
711
+ and every cross-node duration travels as a remaining-duration. `skew` covers bounded clock
712
+ _rate_ divergence over one lease period, not offset. A process suspended past a deadline (VM pause,
713
+ container freeze, long GC) resumes with its monotonic clock advanced and observes its own expiry —
714
+ which is why expiry is checked at commit submission and not only on a timer.
715
+
716
+ ### 6. Drain — recall must revoke capability, not just close the door
717
+
718
+ Two ways a "drained" delegate keeps usable authority, both found in review:
719
+
720
+ - It `save()`s inside an explicit transaction and `unlock()`s before commit; counting live handles
721
+ reports zero while staged writes can still commit inside the lease.
722
+ - It holds a valid `{hold: true}` handle and has staged _nothing_; a recall that only closes new
723
+ admission finds nothing to settle, emits the release, and the holder then writes through its
724
+ still-unexpired handle (`resources/recordLock.ts:32`, `resources/Table.ts:2996`).
725
+
726
+ So recall is defined as four steps, in order:
727
+
728
+ 1. **Close admission** for `K` under this delegation: no new handle, no re-entrant acquisition, no
729
+ `hold` upgrade.
730
+ 2. **Revoke** every outstanding handle's write capability synchronously — the same state an expired
731
+ lease produces, so a subsequent write through it fails 409 at staging _and_ at commit submission.
732
+ Calling `release()` is _not_ revocation: `resources/recordLock.ts:145` deliberately distinguishes a
733
+ handle handed back from a lapsed lease, and `resources/DatabaseTransaction.ts:1213` consults that
734
+ predicate immediately before native submission — so revocation must set the lapsed state, and must
735
+ also reach handles a staged write still holds after `unlock()` removed them from the transaction's
736
+ lock registry. A grace window (`min(remaining delegation, recallGraceMs)`) before revocation is a
737
+ fairness knob, not a safety one.
738
+ 3. **Settle** every transaction that staged a write to `K` under this delegation — committed or
739
+ aborted — including a native commit already submitted, whose completion the pre-submission fence
740
+ does not prove. Settlement follows the **logical** transaction through retry and replay
741
+ (`DatabaseTransaction.ts:1138`, `:1380`); one native attempt completing is not settlement. The
742
+ delegation therefore tracks _staged writes on K_, not live handles, and the release hook hangs off
743
+ transaction settlement.
744
+ 4. **Then** write the release (§7.1).
745
+
746
+ If settlement has not completed by the delegation deadline, no release is written and the successor
747
+ takes the recovery path (§7.2). A failed or rejected recall never fabricates a release (§8). Delegate
748
+ eviction obeys exactly the same rules: a delegate may drop a delegation early, but not while it holds
749
+ an active handle or an unsettled write.
750
+
751
+ ### 7. Freshness
752
+
753
+ #### 7.1 Clean handoff: an inherited dependency set, not a version number
754
+
755
+ The prior Ricart–Agrawala branch got successor freshness free, because a grant rode the grantor's own
756
+ replication stream behind that grantor's data writes. Unicast delegation messages lose that, so it is
757
+ re-established explicitly.
758
+
759
+ After draining, the delegate writes a **`LOCK_RELEASE` control entry** to the table's transaction log
760
+ — the same non-`LOCAL_ONLY` control entry the branch already builds — carrying its inherited
761
+ **dependency set** `{(originNodeName → position)}`. The home merges the release's trusted author at
762
+ the release entry's own position, which is ordered behind that delegate's completed data writes and
763
+ cannot trail the stream cursor. Advancing every releasing holder is conservative when it made no
764
+ write, preserves transitivity without touching the commit hot path, and remains bounded by
765
+ `|members|`.
766
+
767
+ A scalar record version is **not** an applied-history fence, and neither is a single predecessor
768
+ position:
769
+
770
+ - `A` writes `K`; `B` acquires, never writes, releases; `C` has applied `B` but not `A`. A
771
+ `B`-position check passes and `C` reads stale — so the set is **inherited**. B releases `{A:R_A}`;
772
+ the home retains `{A:R_A, B:R_B}`, and C must satisfy both.
773
+ - Core breaks equal-`version` conflicts by node name (`resources/Table.ts:6785`), so `C` can hold a
774
+ _losing_ value at the same timestamp as the winner and pass a `version ≥ V` test. A version scalar
775
+ therefore cannot be the fence even for the simple case.
776
+ - Receiving `B`'s later patch does not prove receipt of `A`'s earlier change to other fields, which
777
+ is the same reason the set cannot be collapsed to its newest member.
778
+
779
+ > **The fence:** admit only when, for every `(origin → position)` in the set, this node has _applied
780
+ > and made visible_ that origin's stream to at least that position. Applied and visible, not received
781
+ > or queued.
782
+
783
+ Deletes and tombstones are writes and carry positions like any other. An origin whose stream has been
784
+ purged past the listed position, or is otherwise unsatisfiable, is a §7.2 recovery case — it must
785
+ fail closed, not silently pass.
786
+
787
+ #### 7.2 Recovery paths, and where the guarantee is explicitly weaker
788
+
789
+ The home holds the dependency set in memory, so a home restart, a generation change, or a delegate
790
+ crash with no clean release loses it. **Open against harper-pro:** whether a durable copy is
791
+ recoverable from the log within retention. If it is, that is a cheaper recovery than the barrier
792
+ below and should be preferred.
793
+
794
+ Without it, the first grant for a key carries a recovery marker, and the acquirer must instead **drain its
795
+ inbound replication streams from every reachable member to the position each held at grant time**
796
+ before admitting. `ClusterLockTransport.establishLockFreshness()` owns this operation, prefers a
797
+ durable release when one remains available, and coalesces concurrent recovery snapshots across keys.
798
+
799
+ **The fence is a `lockBarrier` control entry (harper#2625).** Nothing else on the log can name "the
800
+ position each held at grant time": entries are appended in commit order, not key order (rocksdb-js
801
+ `docs/transaction-log.md`, "Reading The Transaction Log"), so any highest-key head is satisfied on a
802
+ receiver before an earlier-keyed, later-appended entry is applied; a receiver's own received tail is
803
+ zero after its restart while a reachable member still holds committed, undelivered writes; and a
804
+ marker the sender emits at end of stream races `transactionBroadcast`'s `setImmediate`-notified
805
+ queue, so a durable commit can be unobserved when it is emitted. The transport therefore asks each
806
+ reachable member to commit one — `writeLockBarrier(database, table, nonce)` resolves to the entry's
807
+ own log position — and drains that member's stream until it has applied that entry.
808
+
809
+ > **Invariant:** a barrier entry is appended after every transaction the writing node had committed
810
+ > when the barrier was requested, so a peer that has applied that origin's log through the barrier
811
+ > has applied all of them.
812
+
813
+ "Through the barrier" means the barrier entry itself — identified by its origin and position, and by
814
+ the nonce the requesting transport supplied, for the restart case where an origin reissues a clock
815
+ reading — has been applied,
816
+ not that some entry with a key at or past that position has: log keys are not in append order. And
817
+ "applied" is the receiver's contiguous applied-and-visible cursor for that origin, which must not
818
+ advance across an apply failure — core's replicated apply loop logs a terminal commit failure and
819
+ carries on to the next transaction (`resources/Table.ts`, the `beginTxn` backpressure point), so a
820
+ barrier can be present in a receiver's log while a transaction appended before it is absent. The
821
+ entry names no key and no token, replicates exactly as the release does (it is not `LOCAL_ONLY`;
822
+ being replicated is its whole purpose), and the coordinator ignores it on receipt. The transport is
823
+ also handed the wait remaining on the lock's deadline, so it can bound the drain to the lock that
824
+ asked instead of outliving it. **A recovery marker without a barrier fails closed:** a member that
825
+ cannot produce one, or a transport without the operation, leaves the drain with no fence, and
826
+ `lock()` rejects with the retryable 503 below rather than admit on a weaker one.
827
+
828
+ That barrier is the strongest condition available without synchronous replication, and it is
829
+ explicitly weaker in two ways that must be documented rather than implied: an unreachable member's
830
+ committed writes may not be visible, and a predecessor's native commit submitted before expiry can
831
+ settle _after_ the barrier was measured. It is also not cheap — see §10.
832
+
833
+ **Fail-closed is a fallback, not a fix, and the note must not claim otherwise.** If the barrier
834
+ cannot be established, `lock()` rejects with the retryable 503 the branch already uses. But a barrier
835
+ that _succeeds_ still admits the forbidden history: `A` commits `K=1`, becomes unreachable before
836
+ replication, every reachable member's position is satisfied, and `B` is admitted and reads `K=0`.
837
+ Rejecting on failure does not detect that, because nothing failed. Meeting §2 strictly in recovery
838
+ mode requires **recoverable commit evidence** — locked writes replicated to a quorum before the
839
+ critical section is considered settled — which is the third arm of the §7.3 decision, not a free
840
+ default.
841
+
842
+ #### 7.3 The crashed writer: LWW is not enough, and fencing is not free
843
+
844
+ > **Decided: (b) exclusion-only.** (a) and (c) are deferred to harper#2540. §10 carries the
845
+ > normative contract and the reasoning; the analysis below is what those arms would have to build.
846
+
847
+ The earlier claim — that a crashed holder's unreplicated write is stamped older and dropped by
848
+ last-write-wins — does not hold. If `A`'s clock runs ahead, `A` writes under a valid delegation and
849
+ becomes unreachable, and `B` later acquires and commits, `A`'s delayed write can carry the greater
850
+ timestamp and overwrite `B`. Monotonic lease expiry orders admissions, not timestamps.
851
+
852
+ Fencing generations fix it, but the comparison rule has to be **total**, and the obvious pairwise rule
853
+ is not. "Generation for locked writes, timestamp otherwise" cycles: locked `A(g1,t30)`, locked
854
+ `B(g2,t10)`, ordinary `U(t20)` gives `B > A`, `A > U`, `U > B`, and arrival order decides. The two
855
+ coherent options:
856
+
857
+ - **(a) Fenced.** Every write carries a fencing generation (0 when unlocked) and conflict resolution
858
+ is `(generation, timestamp, origin)` lexicographic. Total, no cycle. The costs are real and larger
859
+ than one comparator:
860
+ - It changes `_writeUpdate`'s resolution rule for _all_ writes and adds a field to the record
861
+ contract, so it is a stored-format and older-binary compatibility decision, not just a protocol
862
+ one — including what happens if the feature is disabled after fenced records exist.
863
+ - It **reverses the documented Phase 0 relationship with ordinary writes** (`DESIGN.md:169`): an
864
+ ordinary update landing after a fenced value would permanently lose, regardless of age.
865
+ - Older writes currently enter resequencing and merge logic (`Table.ts:3131`), and deletes take a
866
+ separate path (`Table.ts:3753`), so a generation must be honored by patches and deletes too.
867
+ - A generation alone has no **rejection floor** that survives tombstone cleanup: generation 1 puts,
868
+ generation 2 deletes, the tombstone is reclaimed, and a delayed generation-1 put resurrects the
869
+ record on that replica. Fenced mode therefore also owes a retained floor across tombstone
870
+ reclamation, snapshot copy, restore and replay.
871
+ - **(b) Exclusion-only.** No fencing token in conflict resolution. `lock()` promises exclusion of
872
+ concurrent _admission_; conflict resolution stays last-write-wins, and the crashed-clock-ahead
873
+ overwrite is a documented limitation, as is the §7.2 late-settling commit.
874
+ - **(c) Quorum-confirmed locked writes.** Orthogonal to (a)/(b), and the arm that addresses §2's
875
+ freshness invariant in recovery mode: a locked write is not considered settled until replicated to
876
+ a majority. Write confirmation alone is not sufficient — intersection needs the read side too, and
877
+ §7.2's barrier has no quorum floor — so this arm is **two** changes: confirm the write to a
878
+ majority, _and_ make the barrier require a majority and fail closed below it.
879
+ `X-Replicate-To;confirm=M` (`server/REST.ts:265`) is the nearest existing mechanism, but it is
880
+ super-user-gated and entangled with residency (§10), so this arm cannot just expose it. It costs
881
+ latency on
882
+ every locked write. Even with both halves it does **not** restore §2 as written: the late-settling
883
+ commit had not settled when the barrier ran, so that route needs the commit _fenced_, which is arm
884
+ (a)'s job. §2 as written needs (a) and (c) together.
885
+
886
+ The choice is wider than "fenced or not": it settles the _freshness and settlement_ guarantee as well
887
+ as conflict ordering. §10 records why (b) was taken and states the resulting contract.
888
+
889
+ One limit is outside all three answers: an expired holder can resume work _outside_ Harper while its
890
+ successor runs. `lock()` promises exclusion of valid lock admissions inside Harper; arbitrary
891
+ external effects need idempotency or fencing at that system.
892
+
893
+ ### 8. Failure containment and bounded state
894
+
895
+ - A recall that meets a rejected send, a disconnected peer, a dropped table, a failed release commit,
896
+ a throwing settlement callback or shutdown must **settle its callers and keep admission closed**.
897
+ An escaping rejection can take the process down; swallowing it and granting onward violates §7. A
898
+ failed handoff enters the expiry path and never fabricates a release. These need executable
899
+ contracts — injected synchronous throws and rejections at each of those points — not prose.
900
+ - **Aggregate caps, not just per-key.** The branch's per-key and per-table caps exist to bound what a
901
+ broadcast round accumulates and are removed with it; a scan locking millions of distinct keys would
902
+ otherwise retain millions of delegations, waiters and timers. Bound outstanding delegations per
903
+ database and per requester, bound expiration work per tick, and cancel waiters. Eviction is
904
+ asymmetric, and the asymmetry is the safety rule: a **delegate** may drop a delegation early
905
+ (subject to §6), a **home** may never forget one before its expiry, so its eviction structure
906
+ retains `(key → until)` for expired-but-uncollected entries.
907
+ - **Dependency-set retention is a separate lifetime from delegation expiry.** A home that has evicted
908
+ a key's dependency set cannot distinguish it from a key never delegated at all, and would take the
909
+ §7.2 barrier on both — so a workload cycling through more keys than the cap pays the recovery cost
910
+ during _normal_ operation, not only after a restart. The home therefore keeps a compact
911
+ ever-delegated-in-this-generation filter (add-only, so it has no false negatives): outside it, a key has
912
+ no predecessor and needs no barrier; inside it with the set evicted, the barrier is required. Retain
913
+ dependency sets beyond clean grant removal, and size the filter as part of the cap budget. A grant
914
+ that expires without its exact clean release invalidates any predecessor set for that key and
915
+ forces recovery; retaining it would omit the expired holder from the next fence.
916
+ - Ordinary writes keep their existing ungated path, with no exception: exclusion-only (§7.3) adds
917
+ nothing to a write that was not made under a lock, and delegation bookkeeping lives only on lock
918
+ paths. This is the property the fenced arm would have given up, and it is the main reason it was not
919
+ taken. The requirement on a cached-delegation hit is **zero additional protocol allocation**, not an
920
+ allocation-free `lock()`: Phase 0 itself allocates a handle and a promise, and that is the baseline.
921
+ The implementation does not meet that requirement yet — it allocates an admission record per
922
+ `lock()`. What those records **retain** is bounded, and the bound must be stated exactly rather than
923
+ flatteringly: an admission survives its own `unlock()` on purpose, because the write it staged can
924
+ still commit and the revoker has to stay reachable until the handle's lease runs out (§6 step 2). So
925
+ retention is `lock rate × lease`, which is the floor any correct implementation pays — not a bound
926
+ independent of the lock rate. What the sweep fixes is a different failure: admissions are swept once
927
+ the map has outgrown the live set the previous sweep measured, so mixed lease lengths cannot hide an
928
+ expired admission behind a longer-lived one for the length of _its_ lease. Getting below `rate ×
929
+ lease` means carrying the revoker on the Phase 0 handle that already exists instead of allocating a
930
+ second record, which is the redesign §10's measurement gate is meant to justify.
931
+ - Home-map membership and message role are authenticated and authorized **before** state is
932
+ allocated, so a malformed payload, a superseded generation or cap exhaustion cannot be used to
933
+ accumulate state. A request from a node the current generation does not name is refused before any
934
+ grant, waiter or timer is allocated. Dependency-set and key payload sizes are bounded before
935
+ allocation. Trusted origin identity stays the transport's responsibility, as it already is
936
+ (`resources/recordLockCoordinator.ts:79`). There are no configuration certificates to validate,
937
+ because there is no election — the map's digest is agreed once, out of band, before the feature is
938
+ enabled (§4.1).
939
+
940
+ ### 9. Approaches considered
941
+
942
+ | Axis | Candidate | Why not chosen |
943
+ | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
944
+ | **Different layer** | Put arbitration entirely in harper-pro's replication layer, or behind an embedded/external Raft group that owns the lock table. | The disqualifier for a full replicated lock table is specific, not "bigger": **replicating per-key ownership puts durable, majority-acknowledged work on every acquisition**, which is exactly what volatile delegations remove and what §10's amortization depends on. It would also still need §7's fence, because it is off-stream. The consensus objection is now clean rather than hypocritical: this design runs no election of its own, at any rate, so "adds consensus" is an argument it is entitled to make. If Harper later wants consensus for membership, shard maps and schema changes, §4's map is the piece to replace with it — and the replacement would be invisible to §§5–8. |
945
+ | **Deeper cause** | Do not hand out locks at all: make the conflicting operation a conditional/compare-and-swap write evaluated by one authority (the residency owner). | Covers only retryable single-record read-modify-write. `lock()` exists in harper#483 for a caller holding the lock across arbitrary application work, including calls to other systems, which no CAS expresses. It also does not remove the authority problem: an independent local CAS against asynchronously replicated copies is not cluster-wide exclusion. Worth having _as well_. |
946
+ | **Do less** | Keep Ricart–Agrawala and accept the availability limit: it is implemented, tested, and gated off by `replication.recordLocks`. | The limit is the feature's value, not a rollout caveat: a lock any single unreachable peer disables is not usable for the correctness-critical work `lock()` exists for. Cost points the same way — 13 durable commits and 143 frame deliveries per lock at `P=12`, per key, with no amortization for a node locking the same record repeatedly. |
947
+ | **Do more** | Durable membership epochs: agree `members[]` by single-decree consensus over a majority, with acceptor state persisted before acknowledgement, so an unreachable node is rehomed automatically. **This note's own design through round 6.** | Rejected 2026-09-13. What it buys over the chosen map is exactly one thing: **unattended rehoming**. What it costs is a durable consensus subsystem in harper-pro — persisted promises and accepted values, ballots, renewal leases as the liveness signal, acceptor-side retirement reservations, transitive activation protection, and a conservative restart quarantine to reconstruct reservations a crash lost. All of that exists to _infer_ whether an unreachable node is briefly down or permanently gone. An operator already knows, and can say so in one published generation. The judgement is not that consensus is too hard; it is that **this particular decision has an authority who can simply state it**, so inferring it is unpaid complexity. |
948
+ | **Do more (2)** | A centrally **leased** operator map: the same published generations, but nodes hold short-lived authenticated capabilities for the current one and must renew them from the control plane. The operator stops renewing `g`, waits capability expiry plus the maximum delegation interval, then activates `g+1`. Raised by the round-8 planning review. | **Overruled, with a concrete disqualifier: it makes every node's ability to lock depend on continuously reaching the control plane.** A control-plane outage longer than the capability lease stops record locking on every node in the cluster, including nodes that are healthy and agree with each other — a strictly larger availability dependency than the transition it replaces, and one that is paid continuously rather than at reconfiguration time. It also reintroduces the renewal lease and the clock-rate bound this revision exists to delete, and it does not remove the operator from the loop: the capability issuer must still decide to stop renewing. What it does buy is real: it fences a partitioned old generation mechanically. **Round 10 showed that half can be had without the dependency** — §4.3's one-shot durable activation record, which is issued once per reconfiguration rather than renewed, so steady-state locking never contacts the control plane. That was adopted; only the continuous renewal is rejected. |
949
+ | **Adjacent** | Derive the map from `server.shards`, which already maps shard id → node list and already routes residency reads. | A shard map is a **residency** directive — it says where records live. Making it the lock-home map couples arbitration to data placement and requires sharding configuration, which one customer uses. The chosen map is a separate, purpose-built `recordLockHomes` generation, so a cluster adopts record locks without adopting sharding. Where sharding _is_ configured a shard map may be published as the home set, but only if it already names every node that locks: §4.1's rule binds, so a shard map covering a subset of the cluster would leave the omitted nodes unable to lock at all. |
950
+ | **Chosen** | Operator-agreed immutable home map + rendezvous homes + volatile per-record delegations, local Phase 0 locks underneath. | No agreement protocol runs at any rate: the only agreed state is published out of band and checked by digest. A single arbiter per key removes the contention protocol entirely; a monotonic generation is what makes §5.1's fencing token orderable; repeated locking by one node costs zero cluster messages. The price is stated in §4.2 and is the whole of the tradeoff: **an unavailable home's keys stay unavailable until an operator acts.** |
951
+
952
+ **Quorum-granted bounded delegations** (each voter reserves `K`; `floor(n/2)+1` wins; a preferred
953
+ proposer to damp contention) also amortize local locks, and is the closest rejected alternative. The
954
+ restart argument against it that appeared earlier in this thread was wrong and is withdrawn: a voter
955
+ _can_ forget per-key reservations after outliving every pre-crash vote, given an unexpired
956
+ intersecting quorum. The facts that decide against it are cost and complexity, both concrete: quorum
957
+ fan-out on every delegation acquisition **and renewal**, where a single home pays two unicast
958
+ messages and no renewal fan-out; and a retained contention-arbitration protocol between competing
959
+ claimants, which the single-arbiter design deletes rather than simplifies. What it buys is avoiding
960
+ §4.3's operator-sequenced transition and restart quarantine. For a lock whose stated goal is
961
+ per-record throughput, that trade goes the other way.
962
+
963
+ ### 10. Cost, and the guarantee decision
964
+
965
+ Per uncontended acquisition, `P` participants, `d` = locks served under one delegation:
966
+
967
+ | | durable commits | frame deliveries | latency | one node down |
968
+ | ----------------------------------------------------------------------------------------- | --------------- | ---------------------------------------- | ------------------------------------ | ------------------------------------------------------------------ |
969
+ | Ricart–Agrawala (branch today) | `P+1` | `P²−1` | slowest participant | **all** cluster locks block |
970
+ | This design, first lock on a key | 0 | 2 unicast | 1 RTT to the home, 0 if local | that node's ring share only, until an operator republishes the map |
971
+ | This design, handoff to another node | 1 (release) | `P−1` + 3 unicast | 2 RTT | — |
972
+ | This design, steady state | 0 | 0 | local key lock | — |
973
+ | **Recovery path (§7.2)** — first access per key after a home restart or generation change | 0 | position query to every reachable member | **1 RTT + replication-backlog wait** | — |
974
+
975
+ The recovery row is the one that can dominate and it is not amortized: a scan touching many distinct
976
+ cold keys after a recovery pays it per key. Barriers must be shared or batched across keys without
977
+ weakening their ordering, and the cost measured rather than assumed. Steady-state cost is also
978
+ bounded below by **re-acquisition when a delegation lapses without contention**, so `durationMs` and
979
+ the re-acquisition rate are as load-bearing as the handoff rate.
980
+
981
+ **No measurement exists yet on a real cluster** — not a round's latency, not audit growth, not the
982
+ current `lock()` rate. harper-pro#822 already carries an enablement gate; producing these numbers is
983
+ this design's first deliverable, before the protocol change lands. Everything above is a message
984
+ count, not a benchmark. The gate needs: acquisition latency distribution; delegation re-acquisition
985
+ rate at candidate `durationMs`; the §4.3 restart quarantine's observed effect on lock availability;
986
+ hot-key handoff throughput; **cold-key throughput
987
+ and backlog sensitivity on the recovery path**; allocation rate on the cached-delegation path; cap
988
+ saturation behavior; and throughput with the feature _disabled_, to prove the ungated write path is
989
+ untouched.
990
+
991
+ **The guarantee decision — §7.3.** What `lock()` promises, on two independent axes:
992
+
993
+ | | conflict ordering | recovery-mode freshness | cost |
994
+ | --------------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
995
+ | **Exclusion-only** ✅ | LWW unchanged | narrowed: §2's freshness is not promised on the recovery path | none beyond the protocol |
996
+ | **Fenced** | `(generation, timestamp, origin)` | still narrowed | record-contract + stored-format change, reverses `DESIGN.md:169`, owes a tombstone-surviving rejection floor |
997
+ | **Quorum-confirmed** | either of the above | closes the crashed/unreachable routes, not the late-settling one; §2 as written needs this _and_ fenced | latency on every locked write |
998
+
999
+ **Exclusion-only is chosen.** The other two arms are deferred to harper#2540, which carries the
1000
+ limitations, the costs, and what closing them would take.
1001
+
1002
+ Why the two arms were not taken now: fenced mode's cost is misplaced — it changes conflict resolution for _every_ write in the database,
1003
+ including in deployments that never call `lock()`, and it inverts a rule Phase 0 documented and
1004
+ shipped — in exchange for a predecessor whose write outranks its successor's. Quorum confirmation
1005
+ costs a replication round trip on every locked write. Neither should sit on the critical path of a
1006
+ feature that has not yet been measured.
1007
+
1008
+ > **The contract, and the text the API documentation owes.** `lock()` guarantees that at most one
1009
+ > node **admits** a critical section for a key at a time. It further guarantees that a node admitted
1010
+ > after another node released cleanly has already applied that node's committed writes to the key —
1011
+ > but **only while the key's home still holds the dependency set from that release**. A home restart,
1012
+ > a generation change, or cap eviction (§7.2, §8) loses it and routes the successor to the recovery
1013
+ > barrier instead, which is limitation (2) below: so (2) is reachable after a perfectly clean release,
1014
+ > not only after a failure. It does **not** guarantee either of the following, and neither of them
1015
+ > needs a crash:
1016
+ >
1017
+ > 1. **A predecessor's write can outrank its successor's under last-write-wins.** LWW compares the
1018
+ > transaction timestamp assigned when the write was _staged_ (`resources/Table.ts:3113`), and lease
1019
+ > expiry orders admissions, not timestamps. **`lock()` changes nothing about conflict resolution**
1020
+ > — a locked write and an unlocked write resolve identically, by timestamp, per field, with CRDT
1021
+ > ops folded and the surviving shape depending on both writes' shapes and on whether auditing is
1022
+ > on — so whatever the pair would have done to the record without a lock is what they do with one,
1023
+ > silently and with no error raised. That is the whole of the limitation, and the contract must not
1024
+ > restate the resolution rules: they live in `Table.ts` / `crdt.ts` and are not `lock()`'s to
1025
+ > promise. Two routes in, and they fail differently:
1026
+ > - **(1a) clock skew.** The predecessor's clock ran ahead of its successor's, and its write is
1027
+ > still in flight when the successor writes.
1028
+ > - **(1b) a timestamp pushed ahead on purpose, on a completely clean handoff.** A caller-supplied
1029
+ > future `context.timestamp`, or a mixed explicit transaction whose later timestamp becomes the
1030
+ > handle's floor — both documented as deliberate Phase 0 behavior (`DESIGN.md:305-309`). The
1031
+ > predecessor commits, replicates, drains and releases cleanly; the successor is admitted with
1032
+ > correct freshness, reads the current value, and writes at real wall-clock time — and its write
1033
+ > is the older one, silently, because the stored version is stamped in the future. What the
1034
+ > committed record then holds is whatever ordinary resolution produces from the pair — which for
1035
+ > overlapping plain fields is not the successor's value, and for disjoint patches or commutative
1036
+ > ops may still carry it. No crash, no skew, nothing in flight. **This is the case that
1037
+ > makes "exclusion-only" narrower than it sounds**, and the only defense available to a caller
1038
+ > today is not to stamp future timestamps under a lock.
1039
+ > 2. **Successor freshness is not promised on the recovery path.** The §7.2 barrier drains streams
1040
+ > from every _reachable_ member, so a write the predecessor committed and did not replicate is
1041
+ > invisible to it: the barrier succeeds and the successor reads a stale value. Three routes in, and
1042
+ > only the first is a crash — the predecessor crashed; the predecessor is unreachable; or the
1043
+ > predecessor passed the pre-commit expiry fence, submitted its native commit, and that commit
1044
+ > settles _after_ the barrier was measured. The third needs no failure beyond a commit slower than
1045
+ > the remaining lease. **Two damaged effects on the one record, not two records:** the predecessor's
1046
+ > committed transaction is not reflected in the result, _and_ the successor's own write is computed
1047
+ > from the stale value it read — stored balance 150, successor reads 100 and writes 80, so what
1048
+ > survives is wrong on its own terms and not merely stale. That is exactly the
1049
+ > read-then-conditionally-write the lock exists for.
1050
+ >
1051
+ > **§2's exclusion invariant is wider than what (1) and (2) leave.** It requires a successor to
1052
+ > exclude every predecessor capability that can still admit _or commit_; what ships excludes
1053
+ > admission, and the commit half holds only up to the pre-submission expiry fence
1054
+ > (`resources/DatabaseTransaction.ts:1213`) — a native commit that clears that fence and settles
1055
+ > afterwards is limitation (2)'s third route.
1056
+ >
1057
+ > **There is no caller-side mitigation for (2) — the obvious candidate is unreachable and, where it
1058
+ > is reachable, makes things worse.** `X-Replicate-To` / `confirm=` is meant to be super-user only —
1059
+ > `checkContextPermissions` (`resources/Table.ts:7258`) raises 403 otherwise, though its truthiness
1060
+ > gate lets `X-Replicate-To: 0` through, which is harper#2546 and not a mitigation anyone should
1061
+ > build on. And where it is legitimately available it is a **residency** directive before it is a
1062
+ > confirmation knob: absent a `getResidencyById` function, which short-circuits ahead of it
1063
+ > (`resources/Table.ts:1575`), a numeric value sets residency to `[self, ...N nodes]` and truncates
1064
+ > existing residency on update (`:1574`), while `*` leaves `replicateTo` undefined and falls back to
1065
+ > the database's configured `replication.replicateTo` count (`:1578`, `:642`) — neither of which is
1066
+ > guaranteed to be the whole cluster. In a twelve-node cluster configured `replicateTo: 3`, either
1067
+ > form leaves the record on four nodes, and a successor's barrier over the other eight reachable
1068
+ > members satisfies without any of them ever having held the locked write. That is the opposite of
1069
+ > what the barrier needs.
1070
+ >
1071
+ > Even with residency genuinely cluster-wide, confirmation **narrows (2) and closes none of its three
1072
+ > routes**. For the crashed and unreachable routes it helps only when the confirming and reachable
1073
+ > sets intersect — `M + reachable > |cluster|` — and §7.2's barrier has no read-quorum floor, because
1074
+ > it deliberately tolerates unreachable members; forcing that intersection costs any single down node
1075
+ > blocking every locked write, and `M` is a literal that silently under-confirms after a scale-out.
1076
+ > **For the third route it does nothing at all**: the predecessor's commit had not settled when the
1077
+ > barrier was measured, so no confirmation level makes a barrier taken at `T` observe a commit that
1078
+ > replicates at `T+δ`. Closing (2) needs the barrier to require a majority and fail closed below it,
1079
+ > _and_ the late-settling commit to be fenced rather than confirmed — which is why harper#2540's
1080
+ > quorum-confirmed arm is not a one-line header change and needs the fenced arm with it.
1081
+
1082
+ **Both limitations ship silent, and that is a choice worth recording rather than a consequence.** A
1083
+ caller whose locked write loses gets a 200, no log line and no counter, so nothing distinguishes it
1084
+ from correct behavior. Making it observable is cheap and confined to the lock path — the handle
1085
+ already carries its floor and the commit path already fences per write on `write.lockHandle`
1086
+ (`resources/DatabaseTransaction.ts:1211`), so a lock-path-only check that a staged version exceeds
1087
+ wall-clock, and a counter when a barrier admits with no dependency set, cost nothing on ungated
1088
+ writes. It is not in scope here because it is detection rather than guarantee, but it belongs in
1089
+ harper#2541 rather than nowhere.
1090
+
1091
+ Nothing above may be softened into "rare" or "best-effort" in the API docs, and neither limitation
1092
+ may be described as crash-only or as unreachable on a clean handoff: a documented narrowing is what
1093
+ makes exclusion-only an honest answer rather than an unstated hole, and a narrowing stated more
1094
+ narrowly than it is is the same hole with a paragraph in front of it.
1095
+
1096
+ ### 11. What survives from harper#2498
1097
+
1098
+ Unchanged and reused:
1099
+
1100
+ - `resources/recordLock.ts` — the Phase 0 primitives, the `scope` option, the monotonic lease
1101
+ deadline, the contained release hook.
1102
+ - The commit-time lease fence in `Table.ts` / `DatabaseTransaction.ts`. It moves from a per-round
1103
+ lease to a delegation lease with no change to the mechanism; §6 adds revocation and settlement
1104
+ tracking on top of it rather than replacing it.
1105
+ - `lock()` integration: coalescing of concurrent calls on one key, `scope: 'node'` semantics,
1106
+ fail-closed behavior with no transport, synchronous `unlock()`, the `LockUnavailableError` / 423 /
1107
+ 409 contract.
1108
+ - The `LOCK_RELEASE` control entry end to end — the nibble, the private `Packr`, `recordId: null`
1109
+ and the key in the payload (the audit dedup collision at `ts_R` documented in `DESIGN.md` applies
1110
+ identically), the receive routing off the record path, and the filters on every surface that
1111
+ reports audit entries as record activity.
1112
+ - `ClusterLockTransport` as the boundary, the per-database registry, fail-closed membership
1113
+ validation, and `ownsCoordination()` thread ownership.
1114
+ - harper-pro#822's topology work: participant derivation from the replication group, per-database
1115
+ coordination ownership, the `replication.recordLocks` switch, `cluster_status.recordLocks`.
1116
+
1117
+ **Three defects inherited with that substrate.** A cross-model review of the branch at `b26d5e22`
1118
+ found them in code this note keeps rather than in the arbitration rule it deletes, so they did not go
1119
+ away on their own. **All three are fixed on this branch** as part of landing the replacement; they are
1120
+ recorded here because the reasoning is the design's, not the fix's:
1121
+
1122
+ - **Transport replacement does not fence live authority** (`resources/Table.ts:5474`). Re-registering
1123
+ a transport — a component reload is enough — closes the current coordinator and installs an empty
1124
+ one. `close()` clears coordinator state but does not invalidate handles already handed out, and a
1125
+ handle checks only its own release and lease fields. The successor coordinator can then grant the
1126
+ same key immediately, with no lease time elapsed. Under §5 the equivalent transition is a home
1127
+ restart, and §5's answer applies here too: the replacement must either carry live authority across
1128
+ the swap or fence and settle every outstanding handle before it may grant. Unregister/re-register
1129
+ and transport-object replacement both need coverage.
1130
+ - **The direct receive callback has no containment** (`resources/recordLockCoordinator.ts:908`). The
1131
+ resolver calls `Table.lockCoordinator`, which throws `LockUnavailableError` when the node name is
1132
+ unusable, and that throw happens before `applyEntry()`'s own containment can catch it, so it
1133
+ escapes `deliverLockControlEntry()`. The source-subscription sink already handles this
1134
+ (`resources/Table.ts:884`); the direct callback must too. This is §8's rule — a receive boundary
1135
+ settles its callers and keeps admission closed — applied to a path that exists today.
1136
+ - **The commit fence scans every write on every commit** (`resources/DatabaseTransaction.ts:1213`).
1137
+ The loop runs on ordinary transactions in core-only deployments that never register a transport, so
1138
+ a bulk transaction with 100,000 plain writes pays 100,000 property checks before submission. §8
1139
+ requires ordinary writes to keep their existing ungated path, so the transaction must track whether
1140
+ it holds any lease-protected write and skip the pass when it does not — while still fencing a
1141
+ released-but-staged locked write, which is the case the loop exists for.
1142
+
1143
+ Removed — **done on this branch**:
1144
+
1145
+ - The Ricart–Agrawala state machine — `LOCK_REQUEST`/`LOCK_GRANT` nibbles, per-peer round tracking,
1146
+ deferral queues, `(tsR, nodeName)` ordering, synthesized grants, withdraw-on-timeout, and
1147
+ `agreedDown` DOWN-exclusion. Nibbles 9 and 10 are retired rather than migrated; 9 has since been
1148
+ taken by eviction on `main`, so a migration was never available.
1149
+
1150
+ Added — **core's half is done on this branch**:
1151
+
1152
+ - The home ring (§4.4), the delegation table, recall-and-drain (§6), ordered fencing tokens (§5.1)
1153
+ and §8's aggregate caps, in `resources/recordLockCoordinator.ts`.
1154
+ - The transport interface core needs from harper-pro: `homeMap(database)`,
1155
+ `requestDelegation(...)`, `recallDelegation(...)`, and `establishLockFreshness(...)`, plus the
1156
+ inbound handlers core exposes so a transport can route a peer's request, recall, or positioned
1157
+ release to the right coordinator.
1158
+
1159
+ Added — **harper-pro's delegation server, on harper-pro#822**:
1160
+
1161
+ - The wire: `record_lock_delegate` and `record_lock_recall` as registered operations over the
1162
+ replication connections that already exist, in `replication/recordLockRpc.ts`. A request prefers
1163
+ the live outbound subscription session — whose inbound end already lands on the home's coordinating
1164
+ worker — and falls back to `sendOperationToNode`. The requester's identity is the authenticated
1165
+ node principal of the connection, never a payload field, so a `super_user` human cannot mint or
1166
+ clear a delegation. An operation accepted on a non-owner thread relays through main to the owner
1167
+ worker under a bound, and a relay that times out answers `not-home`, never a grant.
1168
+ - The `ClusterLockTransport` implementation, capability level 2 and the enablement gate, in
1169
+ `replication/recordLockTransport.ts` and `replication/protocolCapabilities.ts`.
1170
+
1171
+ Still owed by harper-pro — **the one thing that blocks enablement**:
1172
+
1173
+ - The operator-agreed home map (§4) behind `transport.homeMap(database)` (harper-pro#825). #822
1174
+ already carries a **static** map as scaffolding, and under this design static is the right shape —
1175
+ what it lacks is the two properties §4.1 requires: an operator-published `recordLockHomes`
1176
+ generation rather than one each node derives for itself, and a digest agreed across peers before
1177
+ grants are enabled — plus §4.3's one-shot activation record, which is what makes a generation change
1178
+ safe rather than merely announced.
1179
+ - **A received `lockRelease` routed to the thread that owns coordination.** Core's log-delivery sink
1180
+ runs wherever `subscribeOnThisThread(applicationWorkerIndex())` is true, which since harper#2524's
1181
+ dedicated application workers is _routinely_ a different thread from the coordinating one — so this
1182
+ is now the expected shape, not an edge case. `applyEntry` drops an entry it receives off the owner
1183
+ thread (it counts and warns, it cannot forward), and the home then holds its grant until
1184
+ `DELEGATION_LEASE_MS + skew` even though the delegate stopped admitting cleanly. Relaying it is the
1185
+ same obligation the delegation RPC already carries.
1186
+ - **The transport registered on every worker thread that can serve a `lock()`**, not only the
1187
+ coordinating one, and including a dedicated application worker (harper#2524). Core cannot check
1188
+ it: the "this database is clustered" latch is per-thread module state, so a worker that never
1189
+ registers never fails closed, and a default-scoped `lock()` there takes the Phase 0 node lock alone
1190
+ while a peer runs the cluster protocol — two nodes admitting one key. Registering everywhere is also
1191
+ what makes the `ownsCoordination()` 503 reachable, which is the path a non-owner worker is supposed
1192
+ to take.
1193
+ - _Addressed in harper-pro#852._ The transport registers from the replication built-in's `start()`,
1194
+ and a built-in is a trusted plugin that `placedOnThisThread` loads on **every** http worker,
1195
+ dedicated application workers (harper#2524) included — so every serving thread latches
1196
+ `clusterRequiredDatabases` and fails closed rather than taking the Phase 0 lock alone. The
1197
+ `ownsCoordination()` 503 is no longer the path a non-owner takes: instead of answering 503 it
1198
+ **relays** the acquire (and the matching release) to the coordinating worker over the worker port
1199
+ mesh, installing the granted admission locally as a remote admission whose handle a recall on the
1200
+ owner fences before the delegation release is written. A `lock()` therefore succeeds uniformly on
1201
+ every http worker at `threads.count > 1`.
1202
+ - **`homeIncarnation` advanced per coordination incarnation, not per process** (§5.1). Core cannot mint
1203
+ it — it must be durable and monotonic — and cannot check it, which puts it in the same class as
1204
+ §4.3's activation record. It matters because coordinator state is per-thread: a replacement
1205
+ coordinating worker restarts the delegation counter at zero, so an incarnation that only advanced
1206
+ per process would let it re-mint tokens its predecessor issued, and a delayed release carrying one
1207
+ of those tokens would clear a live grant. Derived-per-node is why the transport ships gated off: two nodes that disagree
1208
+ derive different rings and can both grant one key. #825 is now a config generation, a digest check
1209
+ and the §4.3 change runbook — not a consensus protocol.
1210
+
1211
+ #### Protocol version and mixed deployments
1212
+
1213
+ The existing `recordLocks` capability does not distinguish Ricart–Agrawala from delegations, and a
1214
+ cluster running both would have two independent arbiters for one key. The capability is therefore
1215
+ **versioned**, the versions are mutually exclusive, and a node advertises exactly one. Since RA never
1216
+ shipped enabled, nibbles 9/10 are retired rather than migrated, and the `LOCK_RELEASE` payload —
1217
+ historically a fixed five-field tuple `[key, requester, generation, homeIncarnation, counter]` — is
1218
+ now `[1, key, requester, generation, homeIncarnation, counter, dependencies]`. The decoder accepts
1219
+ the historical tuple as unknown lineage (therefore recovery), ignores unknown future versions, and
1220
+ still requires an exact fencing-token match before any release can clear a live grant. Harper-pro
1221
+ must advertise a new mutually exclusive capability level for this wire/API contract. The barrier
1222
+ entry (§7.2; nibble 13, payload `[1, nonce]`) rides the same capability: a receiver that predates it
1223
+ resolves the nibble to no entry type at all and the replication sink treats it as an unknown
1224
+ operation, so such a peer must not be sent one — and a recovery drain that reaches a member which
1225
+ cannot produce a barrier has no fence and fails closed.
1226
+
1227
+ **Merging the substrate is itself gated:** nothing that still wires RA arbitration may be reachable
1228
+ as the new protocol.
1229
+
1230
+ ### 12. Verification route
1231
+
1232
+ - **Unit**, coordinator as a pure state machine with **independent per-node clocks** rather than one
1233
+ shared fake clock (`unitTests/resources/recordLockCoordinator.test.js:24` uses a shared one, which
1234
+ cannot express §5.2): delayed grant reply rejected; renewal/restart overlap; recall against a live
1235
+ handle that staged nothing (§6); recall against an unlocked-but-staged write; transitive freshness
1236
+ including the equal-timestamp and dependent-patch cases (§7.1); fencing-token ordering across a home
1237
+ restart (§5.1); and the negative cases — grant inside the §4.3 restart quarantine, a request naming
1238
+ a superseded generation, a grant minted under a generation superseded across the round, a request
1239
+ from a node the current generation does not name, admit before the dependency set is satisfied,
1240
+ admit with an unsatisfiable set.
1241
+ - **Integration, multi-process**, with independent message/apply/commit delays: exclusion and
1242
+ handoff; **a node down and locks still acquired on the majority side**, with a bounded recovery
1243
+ time — the case Ricart–Agrawala cannot pass, and the reason for the change; minority side stops;
1244
+ home crash → operator publishes a new generation → keys re-homed, with the §4.3 drain observed;
1245
+ home restart after a _clean_ release (the §7.2 set-loss path); delegate crash → expiry → successor;
1246
+ native commit delayed past expiry; a generation change that _replaces_ a node rather than removing
1247
+ one; a node holding a superseded generation refused rather than served; mixed protocol versions
1248
+ refuse to interoperate; feature disabled
1249
+ is byte-for-byte the current write path. Counter tests assert observed read/write histories, not
1250
+ only final totals, and snapshot-free reload must still work after the applied-history fence
1251
+ (`Table.ts:2759`). Under exclusion-only there is no fenced-mode suite to add here; the stale
1252
+ writes, patches and deletes against successor records and reclaimed tombstones belong to
1253
+ harper#2540 if that arm is ever taken. What this suite **must** assert instead is the documented
1254
+ narrowing itself, as expected outcomes rather than test failures. Two schedules, and both must
1255
+ assert the observed value rather than only that no error was raised:
1256
+ - **A predecessor's write outranking its successor's**, driven by a caller-supplied future
1257
+ `context.timestamp` rather than a faked clock — so it runs over a _clean_ handoff with no skew.
1258
+ Assert what `lock()` failed to prevent, not a storage disposition: that the committed record
1259
+ differs from what a serialized execution of the two critical sections would have produced, and
1260
+ that no error reached either caller. Choose a fixture where the two writes overlap on a plain
1261
+ field, so the assertion holds under any conflict-resolution change — pinning exact post-resolution
1262
+ values would tie this suite to `Table.ts`/`crdt.ts` behavior that is not `lock()`'s to promise.
1263
+ - **A barrier that succeeds while a predecessor's unreplicated write is invisible**, including the
1264
+ no-crash route where the predecessor's native commit settles after the barrier was measured.
1265
+ Assert both damaged effects on the one record, again over an overlapping plain field: the
1266
+ predecessor's transaction not reflected in the committed result, and the successor's own write
1267
+ computed from the stale value it read.
1268
+ - Failure injection as an executable contract, not prose: synchronous throws and rejected promises
1269
+ through revoke, settlement callbacks, release persistence, shutdown and table removal — callers
1270
+ settle, admission stays closed, no unhandled rejection, no fabricated handoff. A release entry that
1271
+ commits after its waiting caller timed out must be harmless when replayed later.
1272
+ - Enablement stays gated on those results and on §10's measurements.
1273
+
1274
+ ### 13. Rollout
1275
+
1276
+ harper#2498 stays a draft and does not ship Ricart–Agrawala as the arbitration rule. The substrate in
1277
+ §11 lands (reusable under every option considered, and already reviewed); the arbitration **is
1278
+ replaced on the branch**. harper-pro#822 keeps its capability, participant-set, ownership and switch
1279
+ work, and its transport implementation is replaced — the delegation wire is in place there. Neither
1280
+ is enabled by default at any point, and with harper-pro's freshness transport plus harper-pro#825
1281
+ outstanding the branch cannot be enabled even deliberately: core fails closed without either
1282
+ `establishLockFreshness()` or an agreed home map, and no core build supplies them.
1283
+
1284
+ ### 14. Decomposition and implementation status
1285
+
1286
+ Two things had to be settled before implementation, and one still is:
1287
+
1288
+ 1. **The guarantee decision — settled, exclusion-only** (§7.3 (b), §10). The deferred arms are
1289
+ harper#2540.
1290
+ 2. **The measurement gate of §10 comes before the protocol change, not after it.** Tracked as
1291
+ harper-pro#824.
1292
+
1293
+ The work is decomposed as harper-pro#825 (the operator-agreed home map, §4 — a `recordLockHomes`
1294
+ generation, a peer digest check and the §4.3 change runbook), harper#2541 (home ring, delegations,
1295
+ drain and caps, §§5/6/8, plus the three inherited substrate defects in §11) and harper#2542
1296
+ (successor freshness, §7).
1297
+
1298
+ **Status.** harper#2541 and harper#2542's core halves are implemented: the Ricart–Agrawala state machine
1299
+ is gone, and the home ring, the delegation table, recall with §6 steps 1, 2 and 4, ordered fencing
1300
+ tokens, the caps, the §4.3 restart quarantine, the generation monotonicity floor and the
1301
+ transport-swap grant fence are in place, with all three inherited defects fixed. **§6 step 3 — settlement — is not implemented**: a recall revokes capability and then writes
1302
+ the release without waiting for a native commit already submitted to complete. That is exactly
1303
+ limitation (2)'s third route in §10, which the contract already states, so the gap is disclosed
1304
+ rather than hidden — but §6 must not be read as fully implemented. Closing it means hanging the
1305
+ release hook off logical-transaction settlement rather than off the last admission unlocking. What is not here, and is what keeps the feature unusable rather than merely disabled:
1306
+ harper-pro#825's operator-agreed home map — core fails closed without one and no core build supplies
1307
+ one — and harper-pro's `establishLockFreshness()` implementation and positioned release relay. **The
1308
+ measurement gate (harper-pro#824) still has not run**, and the decision to
1309
+ implement ahead of it was the human's, recorded here so the sequence is not mistaken for the one this
1310
+ note recommends.
1311
+
1312
+ One behavior surfaced by implementing §6 is still an open decision, tracked as harper#2580: a
1313
+ `{hold: true}` write that was staged and then unlocked is **revoked** rather than waited for when a
1314
+ successor takes the key, so the request fails at commit. That is safe and it is what §6 specifies,
1315
+ but it is surprising, and the alternative — draining on the transaction rather than the handle —
1316
+ costs bounded-time handoff. If revoke stands, harper#2547 must say so.
1317
+
1318
+ **The documentation obligation is harper#2547, and it is not optional.** §10's rule that the two
1319
+ limitations may not be softened is the condition on which exclusion-only was chosen over harper#2540;
1320
+ if Phase 1 ships behind its gate and the public `lock()` page still describes exclusion without them,
1321
+ the tradeoff that justified the decision was never paid. It lands with or before enablement.
1322
+
1323
+ ## Successor freshness implementation
1324
+
1325
+ Implementation note for harper#2542. The protocol and public guarantee are specified in §§7–8 and
1326
+ §10 of the Phase 1 design above; this note records the implementation boundary and alternatives.
1327
+
1328
+ The stale-read hazard appears when a successor receives a clean grant while one or more predecessor
1329
+ writes are still queued in its inbound replication streams. The home coordinator owns the
1330
+ delegation lineage, so it owns this invariant:
1331
+
1332
+ > Before a successor delegation can admit, its node has applied and made visible every origin-log
1333
+ > position inherited from all prior clean holders of the key; when that lineage is unavailable, it
1334
+ > must complete the explicitly weaker recovery barrier or fail closed.
1335
+
1336
+ ### Chosen implementation
1337
+
1338
+ Core carries a bounded `origin node name -> transaction-log position` dependency set on each held
1339
+ delegation. Clean release serializes the inherited set; when the home applies that release, it merges
1340
+ the release author's own audit-entry position into the set. That position is ordered after every
1341
+ write the holder completed before releasing and, unlike a holder-write timestamp, is necessarily at
1342
+ the head of the author's stream when committed. The home retains the merged set after clearing the
1343
+ grant and returns it with the next grant. Before installing that grant, the requester asks the
1344
+ transport to make every named position applied and visible. Rejection, malformed or unsatisfiable
1345
+ dependencies, and timeout all hand the unclaimed grant back without discarding known lineage and
1346
+ fail the lock with 503.
1347
+
1348
+ The home retains dependency sets independently of live grants, with a larger bounded LRU. A fixed
1349
+ add-only Bloom filter records every key delegated in the current generation. Absence means virgin
1350
+ only when the coordinator has observed the entire generation continuously; cold construction,
1351
+ coordination loss, and generation change saturate that assumption so every unremembered key takes
1352
+ recovery. A continuously observed key absent from the filter needs no barrier; a key present with no
1353
+ retained set (expiry or LRU eviction) receives the recovery marker. The transport resolves that
1354
+ marker by preferring a retained durable release when available, otherwise draining every reachable
1355
+ member to captured positions. It must coalesce concurrent recovery snapshots across keys, and it
1356
+ returns the established positions so the delegation can carry them onward. Bloom false positives
1357
+ only select the slower safe path; there are no false negatives while absence is trusted.
1358
+
1359
+ The recovery drain's fence is the `lockBarrier` control entry (harper#2625). Each reachable member
1360
+ commits one on request — `writeLockBarrier(database, table, nonce)` resolves to the entry's own
1361
+ transaction-log position, and the requester matches the entry on its origin, position and the nonce
1362
+ it supplied — and the requester drains that member's stream until it has applied that entry. The entry is appended after every transaction the member had committed when it was asked,
1363
+ which is the property no highest-key head, received tail, or sender-emitted marker has (§7.2 of the
1364
+ ownership note lists the disqualifier for each). It carries only a version and a nonce, replicates
1365
+ like the release, and the coordinator ignores it. `establishLockFreshness` also receives the wait
1366
+ remaining on the lock deadline so the transport can bound the drain to it. A member or transport
1367
+ that cannot produce a barrier leaves the recovery marker with no fence, and the lock fails closed
1368
+ with the same 503 — the transport's rejection is what fails it: core does not second-guess a
1369
+ positioned recovery reply, and installs the grant on whatever set the transport returns, empty
1370
+ included.
1371
+
1372
+ The release wire format becomes a versioned tuple with a leading version and trailing dependency
1373
+ set. The decoder still accepts the exact historical five-tuple as a release with unknown lineage;
1374
+ an old or malformed entry can clear only the exact live fencing token and forces recovery on the
1375
+ next grant. Unknown future versions are ignored.
1376
+
1377
+ No ordinary or locked-write commit path changes. A cached delegation already passed its barrier and
1378
+ continues to admit locally without a transport call or dependency allocation. Puts, patches, and
1379
+ tombstone deletes are all covered because the release position follows the entire critical section,
1380
+ not because each mutation shape needs its own hook.
1381
+
1382
+ ### Approaches considered
1383
+
1384
+ | Axis | Candidate | Ruling |
1385
+ | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
1386
+ | **Different layer** | Put all lineage and freshness state in harper-pro's replication transport. | Rejected: core alone owns delegation grant/release lineage and the exact-token rule. Moving the per-key state outward would duplicate that state and permit coordinator and transport to disagree about which release advanced a key. The transport does own the stream-specific apply-visible wait, so the chosen boundary exposes only that operation. |
1387
+ | **Deeper cause** | Use only the immediate predecessor's replicated release entry as the freshness fence. | Rejected: release ordering covers only the releasing node's own stream. In `A writes -> B inherits but does not write -> C`, observing B's release says nothing about whether C has applied A's write; the release position must be merged into an inherited transitive set. |
1388
+ | **Do less** | Carry one scalar record version or wait only for the immediate predecessor. | Rejected: record resolution versions are not origin-log positions, equal timestamps from different origins do not identify stream progress, and a dependent patch can require more than the winning record version. Immediate-predecessor waiting also fails the no-write B handoff above. |
1389
+ | **Chosen** | Core-owned inherited dependency sets, advanced by the trusted release-entry position, plus transport-owned apply-visible and coalesced recovery barriers. | This keeps delegation lineage in its owner, expresses progress in coordinates replication can actually wait on, preserves transitivity, leaves cached acquisitions and all commit paths unchanged, and avoids holder timestamps that may sort behind an already-advanced replication cursor. |
1390
+
1391
+ The release-entry position replaces an earlier per-write commit-hook design. It removes a hot-path
1392
+ callback, covers every write that remains authorized when the release is committed, and cannot sort
1393
+ behind a replication cursor that has already advanced. Cold-state trust is discarded whenever
1394
+ history may have been missed, and pending grants remain recallable before admission.
1395
+
1396
+ ### Applied-prefix holes
1397
+
1398
+ An origin's committed `lockBarrier` proves its preceding prefix only up to holes reported to the
1399
+ transport. A later barrier cannot turn a terminally failed and skipped replicated transaction
1400
+ into an applied write. Core's `registerReplicatedApplyFailureListener(database, listener)` reports
1401
+ the failed audit-header origin `nodeId` and origin transaction-log `position`, and awaits every
1402
+ registered listener before pulling another event (or staging the new `beginTxn` that closed the
1403
+ failed transaction). The transport must durably record that discontinuity before its listener
1404
+ resolves and reject freshness proofs crossing it until its own recovery rule clears the hole.
1405
+ Registration is per apply worker; listener failures remain log-and-continue, so the transport
1406
+ also owns failing closed when it cannot persist hole state. Core supplies the observation and
1407
+ ordering hook; durable poison records and their clearing policy belong to the transport.