@harperfast/harper 5.2.10 → 5.3.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (554) hide show
  1. package/agent/mcpTools.ts +1 -1
  2. package/agent/session.ts +25 -14
  3. package/bin/cliOperations.ts +46 -9
  4. package/bin/copyDb.ts +282 -59
  5. package/bin/deploySetup.ts +16 -5
  6. package/bin/harper.ts +1 -1
  7. package/bin/help.ts +4 -1
  8. package/bin/lite.ts +4 -1
  9. package/bin/restart.ts +59 -3
  10. package/bin/run.ts +6 -11
  11. package/bin/upgrade.js +7 -3
  12. package/bin/workloadIdentity.ts +119 -0
  13. package/components/Application.ts +2414 -242
  14. package/components/ApplicationScope.ts +8 -0
  15. package/components/EntryHandler.ts +59 -39
  16. package/components/OptionsWatcher.ts +150 -74
  17. package/components/RuntimeModuleTracker.ts +38 -7
  18. package/components/Scope.ts +37 -15
  19. package/components/awaitRestart.ts +84 -0
  20. package/components/componentLoader.ts +340 -30
  21. package/components/componentPreparationLock.ts +16 -5
  22. package/components/mcp/adapters/harperHttp.ts +4 -0
  23. package/components/mcp/listChanged.ts +4 -0
  24. package/components/mcp/toolRegistry.ts +2 -0
  25. package/components/mcp/tools/operations.ts +9 -0
  26. package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
  27. package/components/operations.js +270 -78
  28. package/components/operationsValidation.js +49 -1
  29. package/components/status/ComponentStatusRegistry.ts +59 -0
  30. package/config/RootConfigWatcher.ts +80 -34
  31. package/config/configUtils.ts +291 -34
  32. package/config/harperConfigEnvVars.ts +170 -27
  33. package/config-root.schema.json +29 -0
  34. package/dataLayer/blobBackup.ts +160 -50
  35. package/dataLayer/delete.ts +6 -1
  36. package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
  37. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  38. package/dataLayer/hdbInfoController.ts +34 -1
  39. package/dataLayer/insert.ts +44 -1
  40. package/dataLayer/rocksdbBackup.ts +53 -10
  41. package/dataLayer/schema.ts +11 -1
  42. package/dataLayer/schemaDescribe.ts +8 -1
  43. package/dist/agent/mcpTools.js +1 -1
  44. package/dist/agent/mcpTools.js.map +1 -1
  45. package/dist/agent/session.d.ts +22 -0
  46. package/dist/agent/session.js +26 -15
  47. package/dist/agent/session.js.map +1 -1
  48. package/dist/bin/cliOperations.js +48 -9
  49. package/dist/bin/cliOperations.js.map +1 -1
  50. package/dist/bin/copyDb.d.ts +12 -1
  51. package/dist/bin/copyDb.js +248 -60
  52. package/dist/bin/copyDb.js.map +1 -1
  53. package/dist/bin/deploySetup.d.ts +2 -0
  54. package/dist/bin/deploySetup.js +11 -3
  55. package/dist/bin/deploySetup.js.map +1 -1
  56. package/dist/bin/harper.js +1 -1
  57. package/dist/bin/harper.js.map +1 -1
  58. package/dist/bin/help.js +4 -1
  59. package/dist/bin/help.js.map +1 -1
  60. package/dist/bin/lite.js +4 -1
  61. package/dist/bin/lite.js.map +1 -1
  62. package/dist/bin/restart.js +54 -5
  63. package/dist/bin/restart.js.map +1 -1
  64. package/dist/bin/run.js +4 -10
  65. package/dist/bin/run.js.map +1 -1
  66. package/dist/bin/upgrade.js +4 -3
  67. package/dist/bin/upgrade.js.map +1 -1
  68. package/dist/bin/workloadIdentity.d.ts +18 -0
  69. package/dist/bin/workloadIdentity.js +100 -0
  70. package/dist/bin/workloadIdentity.js.map +1 -0
  71. package/dist/components/Application.d.ts +139 -16
  72. package/dist/components/Application.js +2215 -267
  73. package/dist/components/Application.js.map +1 -1
  74. package/dist/components/ApplicationScope.d.ts +8 -0
  75. package/dist/components/ApplicationScope.js +7 -0
  76. package/dist/components/ApplicationScope.js.map +1 -1
  77. package/dist/components/EntryHandler.js +26 -10
  78. package/dist/components/EntryHandler.js.map +1 -1
  79. package/dist/components/OptionsWatcher.d.ts +1 -0
  80. package/dist/components/OptionsWatcher.js +141 -74
  81. package/dist/components/OptionsWatcher.js.map +1 -1
  82. package/dist/components/RuntimeModuleTracker.js +40 -6
  83. package/dist/components/RuntimeModuleTracker.js.map +1 -1
  84. package/dist/components/Scope.js +38 -13
  85. package/dist/components/Scope.js.map +1 -1
  86. package/dist/components/awaitRestart.d.ts +33 -0
  87. package/dist/components/awaitRestart.js +61 -0
  88. package/dist/components/awaitRestart.js.map +1 -0
  89. package/dist/components/componentLoader.d.ts +38 -1
  90. package/dist/components/componentLoader.js +279 -22
  91. package/dist/components/componentLoader.js.map +1 -1
  92. package/dist/components/componentPreparationLock.d.ts +5 -0
  93. package/dist/components/componentPreparationLock.js +14 -6
  94. package/dist/components/componentPreparationLock.js.map +1 -1
  95. package/dist/components/mcp/adapters/harperHttp.js +4 -0
  96. package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
  97. package/dist/components/mcp/listChanged.js +5 -0
  98. package/dist/components/mcp/listChanged.js.map +1 -1
  99. package/dist/components/mcp/toolRegistry.d.ts +1 -0
  100. package/dist/components/mcp/toolRegistry.js.map +1 -1
  101. package/dist/components/mcp/tools/operations.d.ts +5 -0
  102. package/dist/components/mcp/tools/operations.js +9 -0
  103. package/dist/components/mcp/tools/operations.js.map +1 -1
  104. package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
  105. package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
  106. package/dist/components/operations.js +231 -77
  107. package/dist/components/operations.js.map +1 -1
  108. package/dist/components/operationsValidation.js +49 -1
  109. package/dist/components/operationsValidation.js.map +1 -1
  110. package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
  111. package/dist/components/status/ComponentStatusRegistry.js +63 -0
  112. package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
  113. package/dist/config/RootConfigWatcher.d.ts +7 -1
  114. package/dist/config/RootConfigWatcher.js +64 -27
  115. package/dist/config/RootConfigWatcher.js.map +1 -1
  116. package/dist/config/configUtils.d.ts +9 -1
  117. package/dist/config/configUtils.js +254 -33
  118. package/dist/config/configUtils.js.map +1 -1
  119. package/dist/config/harperConfigEnvVars.d.ts +16 -0
  120. package/dist/config/harperConfigEnvVars.js +162 -25
  121. package/dist/config/harperConfigEnvVars.js.map +1 -1
  122. package/dist/dataLayer/blobBackup.d.ts +49 -20
  123. package/dist/dataLayer/blobBackup.js +139 -50
  124. package/dist/dataLayer/blobBackup.js.map +1 -1
  125. package/dist/dataLayer/delete.js +1 -1
  126. package/dist/dataLayer/delete.js.map +1 -1
  127. package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
  128. package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
  129. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  130. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
  131. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  132. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
  133. package/dist/dataLayer/hdbInfoController.d.ts +10 -0
  134. package/dist/dataLayer/hdbInfoController.js +30 -1
  135. package/dist/dataLayer/hdbInfoController.js.map +1 -1
  136. package/dist/dataLayer/insert.d.ts +9 -1
  137. package/dist/dataLayer/insert.js +30 -0
  138. package/dist/dataLayer/insert.js.map +1 -1
  139. package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
  140. package/dist/dataLayer/rocksdbBackup.js +45 -8
  141. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  142. package/dist/dataLayer/schema.js +8 -0
  143. package/dist/dataLayer/schema.js.map +1 -1
  144. package/dist/dataLayer/schemaDescribe.js +8 -1
  145. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  146. package/dist/index.d.ts +1 -1
  147. package/dist/index.js.map +1 -1
  148. package/dist/json/systemSchema.json +52 -0
  149. package/dist/resources/DatabaseTransaction.d.ts +55 -3
  150. package/dist/resources/DatabaseTransaction.js +460 -134
  151. package/dist/resources/DatabaseTransaction.js.map +1 -1
  152. package/dist/resources/LMDBTransaction.js +22 -4
  153. package/dist/resources/LMDBTransaction.js.map +1 -1
  154. package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
  155. package/dist/resources/PrimaryRocksDatabase.js +30 -2
  156. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  157. package/dist/resources/RecordEncoder.d.ts +20 -0
  158. package/dist/resources/RecordEncoder.js +94 -9
  159. package/dist/resources/RecordEncoder.js.map +1 -1
  160. package/dist/resources/RequestTarget.d.ts +2 -0
  161. package/dist/resources/RequestTarget.js.map +1 -1
  162. package/dist/resources/Resource.js +20 -11
  163. package/dist/resources/Resource.js.map +1 -1
  164. package/dist/resources/ResourceInterface.d.ts +20 -1
  165. package/dist/resources/ResourceInterface.js.map +1 -1
  166. package/dist/resources/RocksIndexStore.d.ts +6 -1
  167. package/dist/resources/RocksIndexStore.js +24 -10
  168. package/dist/resources/RocksIndexStore.js.map +1 -1
  169. package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
  170. package/dist/resources/RocksTransactionLogStore.js +57 -17
  171. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  172. package/dist/resources/Table.d.ts +112 -8
  173. package/dist/resources/Table.js +1071 -197
  174. package/dist/resources/Table.js.map +1 -1
  175. package/dist/resources/auditStore.d.ts +11 -2
  176. package/dist/resources/auditStore.js +183 -18
  177. package/dist/resources/auditStore.js.map +1 -1
  178. package/dist/resources/blob.d.ts +129 -9
  179. package/dist/resources/blob.js +992 -114
  180. package/dist/resources/blob.js.map +1 -1
  181. package/dist/resources/branchDatabase.d.ts +48 -0
  182. package/dist/resources/branchDatabase.js +892 -0
  183. package/dist/resources/branchDatabase.js.map +1 -0
  184. package/dist/resources/crdt.js +50 -12
  185. package/dist/resources/crdt.js.map +1 -1
  186. package/dist/resources/databases.d.ts +150 -9
  187. package/dist/resources/databases.js +1247 -525
  188. package/dist/resources/databases.js.map +1 -1
  189. package/dist/resources/defineTable.d.ts +10 -2
  190. package/dist/resources/defineTable.js +9 -1
  191. package/dist/resources/defineTable.js.map +1 -1
  192. package/dist/resources/graphql.d.ts +1 -1
  193. package/dist/resources/graphql.js +50 -15
  194. package/dist/resources/graphql.js.map +1 -1
  195. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
  196. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
  197. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  198. package/dist/resources/longLivedTransactions.d.ts +71 -0
  199. package/dist/resources/longLivedTransactions.js +358 -0
  200. package/dist/resources/longLivedTransactions.js.map +1 -0
  201. package/dist/resources/models/backendRegistry.d.ts +26 -0
  202. package/dist/resources/models/backendRegistry.js +60 -2
  203. package/dist/resources/models/backendRegistry.js.map +1 -1
  204. package/dist/resources/models/bootstrap.d.ts +33 -1
  205. package/dist/resources/models/bootstrap.js +416 -31
  206. package/dist/resources/models/bootstrap.js.map +1 -1
  207. package/dist/resources/recordLock.d.ts +80 -0
  208. package/dist/resources/recordLock.js +184 -0
  209. package/dist/resources/recordLock.js.map +1 -0
  210. package/dist/resources/replayLogs.d.ts +14 -1
  211. package/dist/resources/replayLogs.js +152 -24
  212. package/dist/resources/replayLogs.js.map +1 -1
  213. package/dist/resources/replayLogsGuards.d.ts +94 -7
  214. package/dist/resources/replayLogsGuards.js +111 -7
  215. package/dist/resources/replayLogsGuards.js.map +1 -1
  216. package/dist/resources/search.js +158 -32
  217. package/dist/resources/search.js.map +1 -1
  218. package/dist/resources/transactionBroadcast.d.ts +1 -1
  219. package/dist/resources/transactionBroadcast.js +2 -2
  220. package/dist/resources/transactionBroadcast.js.map +1 -1
  221. package/dist/security/auth.js +61 -30
  222. package/dist/security/auth.js.map +1 -1
  223. package/dist/security/authn/oidc/claims.d.ts +22 -0
  224. package/dist/security/authn/oidc/claims.js +71 -0
  225. package/dist/security/authn/oidc/claims.js.map +1 -0
  226. package/dist/security/authn/oidc/identityToken.d.ts +27 -0
  227. package/dist/security/authn/oidc/identityToken.js +111 -0
  228. package/dist/security/authn/oidc/identityToken.js.map +1 -0
  229. package/dist/security/authn/oidc/jwks.d.ts +25 -0
  230. package/dist/security/authn/oidc/jwks.js +261 -0
  231. package/dist/security/authn/oidc/jwks.js.map +1 -0
  232. package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
  233. package/dist/security/authn/oidc/providers/generic.js +34 -0
  234. package/dist/security/authn/oidc/providers/generic.js.map +1 -0
  235. package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
  236. package/dist/security/authn/oidc/providers/githubActions.js +129 -0
  237. package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
  238. package/dist/security/authn/oidc/providers/index.d.ts +37 -0
  239. package/dist/security/authn/oidc/providers/index.js +24 -0
  240. package/dist/security/authn/oidc/providers/index.js.map +1 -0
  241. package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
  242. package/dist/security/authn/oidc/tokenExchange.js +306 -0
  243. package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
  244. package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
  245. package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
  246. package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
  247. package/dist/security/authn/oidc/types.d.ts +38 -0
  248. package/dist/security/authn/oidc/types.js +6 -0
  249. package/dist/security/authn/oidc/types.js.map +1 -0
  250. package/dist/security/certificateVerification/index.js +40 -11
  251. package/dist/security/certificateVerification/index.js.map +1 -1
  252. package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
  253. package/dist/security/certificateVerification/trustedIssuers.js +79 -0
  254. package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
  255. package/dist/security/certificateVerification/types.d.ts +1 -0
  256. package/dist/security/credentialProvenance.d.ts +35 -0
  257. package/dist/security/credentialProvenance.js +51 -0
  258. package/dist/security/credentialProvenance.js.map +1 -0
  259. package/dist/security/credentialRejection.d.ts +4 -0
  260. package/dist/security/credentialRejection.js +24 -0
  261. package/dist/security/credentialRejection.js.map +1 -0
  262. package/dist/security/deferredAuthentication.d.ts +36 -0
  263. package/dist/security/deferredAuthentication.js +70 -0
  264. package/dist/security/deferredAuthentication.js.map +1 -0
  265. package/dist/security/impersonation.d.ts +21 -0
  266. package/dist/security/impersonation.js +108 -9
  267. package/dist/security/impersonation.js.map +1 -1
  268. package/dist/security/jsLoader.d.ts +6 -0
  269. package/dist/security/jsLoader.js +75 -15
  270. package/dist/security/jsLoader.js.map +1 -1
  271. package/dist/security/keys.js +301 -71
  272. package/dist/security/keys.js.map +1 -1
  273. package/dist/security/operationScope.d.ts +21 -0
  274. package/dist/security/operationScope.js +36 -0
  275. package/dist/security/operationScope.js.map +1 -0
  276. package/dist/security/permissionsTranslator.js +21 -0
  277. package/dist/security/permissionsTranslator.js.map +1 -1
  278. package/dist/security/tokenAuthentication.d.ts +19 -1
  279. package/dist/security/tokenAuthentication.js +191 -10
  280. package/dist/security/tokenAuthentication.js.map +1 -1
  281. package/dist/security/user.js +4 -3
  282. package/dist/security/user.js.map +1 -1
  283. package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
  284. package/dist/server/DurableSubscriptionsSession.js +67 -10
  285. package/dist/server/DurableSubscriptionsSession.js.map +1 -1
  286. package/dist/server/REST.js +73 -0
  287. package/dist/server/REST.js.map +1 -1
  288. package/dist/server/graphqlQuerying.js +4 -0
  289. package/dist/server/graphqlQuerying.js.map +1 -1
  290. package/dist/server/http.d.ts +11 -0
  291. package/dist/server/http.js +88 -15
  292. package/dist/server/http.js.map +1 -1
  293. package/dist/server/itc/serverHandlers.js +8 -1
  294. package/dist/server/itc/serverHandlers.js.map +1 -1
  295. package/dist/server/jobs/jobProcess.js +6 -2
  296. package/dist/server/jobs/jobProcess.js.map +1 -1
  297. package/dist/server/jobs/jobs.js +4 -1
  298. package/dist/server/jobs/jobs.js.map +1 -1
  299. package/dist/server/liveSubscriptionAuth.d.ts +26 -4
  300. package/dist/server/liveSubscriptionAuth.js +105 -39
  301. package/dist/server/liveSubscriptionAuth.js.map +1 -1
  302. package/dist/server/loadRootComponents.js +49 -10
  303. package/dist/server/loadRootComponents.js.map +1 -1
  304. package/dist/server/mqtt.d.ts +2 -0
  305. package/dist/server/mqtt.js +165 -30
  306. package/dist/server/mqtt.js.map +1 -1
  307. package/dist/server/nodeName.d.ts +2 -0
  308. package/dist/server/nodeName.js +107 -23
  309. package/dist/server/nodeName.js.map +1 -1
  310. package/dist/server/serverHelpers/Headers.d.ts +25 -0
  311. package/dist/server/serverHelpers/Headers.js +139 -1
  312. package/dist/server/serverHelpers/Headers.js.map +1 -1
  313. package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
  314. package/dist/server/serverHelpers/contentTypes.js +32 -25
  315. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  316. package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
  317. package/dist/server/serverHelpers/deployValidationState.js +9 -19
  318. package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
  319. package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
  320. package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
  321. package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
  322. package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
  323. package/dist/server/serverHelpers/registeredOperations.js +74 -21
  324. package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
  325. package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
  326. package/dist/server/serverHelpers/requestSanitization.js +20 -0
  327. package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
  328. package/dist/server/serverHelpers/serverHandlers.js +6 -3
  329. package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
  330. package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
  331. package/dist/server/serverHelpers/serverUtilities.js +90 -20
  332. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  333. package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
  334. package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
  335. package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
  336. package/dist/server/serverHelpers/uwsServer.js +19 -1
  337. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  338. package/dist/server/static.js +24 -28
  339. package/dist/server/static.js.map +1 -1
  340. package/dist/server/storageReclamation.d.ts +5 -0
  341. package/dist/server/storageReclamation.js +16 -0
  342. package/dist/server/storageReclamation.js.map +1 -1
  343. package/dist/server/threads/itc.d.ts +7 -2
  344. package/dist/server/threads/itc.js +5 -1
  345. package/dist/server/threads/itc.js.map +1 -1
  346. package/dist/server/threads/manageThreads.d.ts +26 -4
  347. package/dist/server/threads/manageThreads.js +517 -72
  348. package/dist/server/threads/manageThreads.js.map +1 -1
  349. package/dist/server/threads/socketRouter.js +89 -26
  350. package/dist/server/threads/socketRouter.js.map +1 -1
  351. package/dist/server/threads/threadHeapMemory.d.ts +2 -0
  352. package/dist/server/threads/threadHeapMemory.js +31 -0
  353. package/dist/server/threads/threadHeapMemory.js.map +1 -0
  354. package/dist/server/threads/threadServer.js +46 -15
  355. package/dist/server/threads/threadServer.js.map +1 -1
  356. package/dist/sqlEngine/config.d.ts +1 -3
  357. package/dist/sqlEngine/config.js +19 -16
  358. package/dist/sqlEngine/config.js.map +1 -1
  359. package/dist/sqlTranslator/index.d.ts +1 -1
  360. package/dist/sqlTranslator/index.js +30 -7
  361. package/dist/sqlTranslator/index.js.map +1 -1
  362. package/dist/upgrade/directives/5-3-0.d.ts +7 -0
  363. package/dist/upgrade/directives/5-3-0.js +148 -0
  364. package/dist/upgrade/directives/5-3-0.js.map +1 -0
  365. package/dist/upgrade/directives/directivesController.js +2 -1
  366. package/dist/upgrade/directives/directivesController.js.map +1 -1
  367. package/dist/utility/OperationFunctionCaller.js +2 -1
  368. package/dist/utility/OperationFunctionCaller.js.map +1 -1
  369. package/dist/utility/common_utils.d.ts +16 -0
  370. package/dist/utility/common_utils.js +32 -6
  371. package/dist/utility/common_utils.js.map +1 -1
  372. package/dist/utility/componentNames.d.ts +8 -0
  373. package/dist/utility/componentNames.js +12 -1
  374. package/dist/utility/componentNames.js.map +1 -1
  375. package/dist/utility/environment/environmentManager.js +3 -6
  376. package/dist/utility/environment/environmentManager.js.map +1 -1
  377. package/dist/utility/environment/systemInformation.d.ts +1 -0
  378. package/dist/utility/environment/systemInformation.js +1 -0
  379. package/dist/utility/environment/systemInformation.js.map +1 -1
  380. package/dist/utility/errors/commonErrors.d.ts +2 -0
  381. package/dist/utility/errors/commonErrors.js +2 -0
  382. package/dist/utility/errors/commonErrors.js.map +1 -1
  383. package/dist/utility/errors/hdbError.d.ts +17 -0
  384. package/dist/utility/errors/hdbError.js +30 -1
  385. package/dist/utility/errors/hdbError.js.map +1 -1
  386. package/dist/utility/globalSchema.d.ts +18 -0
  387. package/dist/utility/hdbTerms.d.ts +16 -0
  388. package/dist/utility/hdbTerms.js +18 -2
  389. package/dist/utility/hdbTerms.js.map +1 -1
  390. package/dist/utility/logging/harper_logger.d.ts +2 -0
  391. package/dist/utility/logging/harper_logger.js +91 -14
  392. package/dist/utility/logging/harper_logger.js.map +1 -1
  393. package/dist/utility/logging/logRotator.js +76 -49
  394. package/dist/utility/logging/logRotator.js.map +1 -1
  395. package/dist/utility/nodeIdentity.d.ts +9 -0
  396. package/dist/utility/nodeIdentity.js +58 -0
  397. package/dist/utility/nodeIdentity.js.map +1 -0
  398. package/dist/utility/npmUtilities.js +9 -7
  399. package/dist/utility/npmUtilities.js.map +1 -1
  400. package/dist/utility/operationPermissions.d.ts +3 -1
  401. package/dist/utility/operationPermissions.js +16 -1
  402. package/dist/utility/operationPermissions.js.map +1 -1
  403. package/dist/utility/operation_authorization.d.ts +10 -7
  404. package/dist/utility/operation_authorization.js +212 -41
  405. package/dist/utility/operation_authorization.js.map +1 -1
  406. package/dist/utility/watchPath.d.ts +29 -0
  407. package/dist/utility/watchPath.js +68 -0
  408. package/dist/utility/watchPath.js.map +1 -0
  409. package/dist/utility/watcherFallback.d.ts +86 -0
  410. package/dist/utility/watcherFallback.js +278 -1
  411. package/dist/utility/watcherFallback.js.map +1 -1
  412. package/dist/validation/configValidator.d.ts +12 -0
  413. package/dist/validation/configValidator.js +199 -72
  414. package/dist/validation/configValidator.js.map +1 -1
  415. package/dist/validation/installValidator.js +12 -0
  416. package/dist/validation/installValidator.js.map +1 -1
  417. package/dist/validation/validationWrapper.d.ts +11 -0
  418. package/dist/validation/validationWrapper.js +16 -3
  419. package/dist/validation/validationWrapper.js.map +1 -1
  420. package/index.ts +2 -0
  421. package/json/systemSchema.json +52 -0
  422. package/npm-shrinkwrap.json +235 -233
  423. package/package.json +8 -7
  424. package/resources/DESIGN.md +114 -52
  425. package/resources/DatabaseTransaction.ts +522 -130
  426. package/resources/LMDBTransaction.ts +26 -4
  427. package/resources/PrimaryRocksDatabase.ts +29 -3
  428. package/resources/RecordEncoder.ts +103 -8
  429. package/resources/RequestTarget.ts +2 -0
  430. package/resources/Resource.ts +17 -9
  431. package/resources/ResourceInterface.ts +23 -0
  432. package/resources/RocksIndexStore.ts +30 -10
  433. package/resources/RocksTransactionLogStore.ts +78 -21
  434. package/resources/Table.ts +1107 -164
  435. package/resources/auditStore.ts +187 -21
  436. package/resources/blob.ts +1029 -110
  437. package/resources/branchDatabase.ts +941 -0
  438. package/resources/crdt.ts +76 -12
  439. package/resources/databases.ts +1354 -522
  440. package/resources/defineTable.ts +18 -2
  441. package/resources/graphql.ts +70 -16
  442. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
  443. package/resources/longLivedTransactions.ts +360 -0
  444. package/resources/models/backendRegistry.ts +84 -2
  445. package/resources/models/bootstrap.ts +473 -28
  446. package/resources/recordLock.ts +253 -0
  447. package/resources/replayLogs.ts +147 -26
  448. package/resources/replayLogsGuards.ts +171 -8
  449. package/resources/search.ts +154 -32
  450. package/resources/transactionBroadcast.ts +3 -3
  451. package/security/auth.ts +68 -29
  452. package/security/authn/oidc/claims.ts +72 -0
  453. package/security/authn/oidc/identityToken.ts +129 -0
  454. package/security/authn/oidc/jwks.ts +260 -0
  455. package/security/authn/oidc/providers/generic.ts +40 -0
  456. package/security/authn/oidc/providers/githubActions.ts +137 -0
  457. package/security/authn/oidc/providers/index.ts +52 -0
  458. package/security/authn/oidc/tokenExchange.ts +300 -0
  459. package/security/authn/oidc/trustPolicyOperations.ts +343 -0
  460. package/security/authn/oidc/types.ts +41 -0
  461. package/security/certificateVerification/index.ts +54 -13
  462. package/security/certificateVerification/trustedIssuers.ts +76 -0
  463. package/security/certificateVerification/types.ts +1 -0
  464. package/security/credentialProvenance.ts +47 -0
  465. package/security/credentialRejection.ts +22 -0
  466. package/security/deferredAuthentication.ts +71 -0
  467. package/security/impersonation.ts +117 -12
  468. package/security/jsLoader.ts +81 -18
  469. package/security/keys.ts +298 -72
  470. package/security/operationScope.ts +33 -0
  471. package/security/permissionsTranslator.js +23 -0
  472. package/security/tokenAuthentication.ts +233 -12
  473. package/security/user.ts +4 -3
  474. package/server/DESIGN.md +183 -16
  475. package/server/DurableSubscriptionsSession.ts +71 -11
  476. package/server/REST.ts +79 -1
  477. package/server/graphqlQuerying.ts +4 -0
  478. package/server/http.ts +99 -18
  479. package/server/itc/serverHandlers.js +8 -1
  480. package/server/jobs/jobProcess.ts +8 -2
  481. package/server/jobs/jobs.ts +4 -1
  482. package/server/liveSubscriptionAuth.ts +129 -46
  483. package/server/loadRootComponents.js +50 -8
  484. package/server/mqtt.ts +179 -38
  485. package/server/nodeName.ts +103 -21
  486. package/server/serverHelpers/Headers.ts +131 -0
  487. package/server/serverHelpers/contentTypes.ts +29 -21
  488. package/server/serverHelpers/deployValidationState.ts +24 -13
  489. package/server/serverHelpers/operationAuthorizationState.ts +34 -3
  490. package/server/serverHelpers/registeredOperations.ts +79 -22
  491. package/server/serverHelpers/requestSanitization.ts +15 -0
  492. package/server/serverHelpers/serverHandlers.js +6 -3
  493. package/server/serverHelpers/serverUtilities.ts +104 -31
  494. package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
  495. package/server/serverHelpers/uwsServer.ts +17 -2
  496. package/server/static.ts +23 -29
  497. package/server/storageReclamation.ts +13 -0
  498. package/server/threads/itc.js +11 -1
  499. package/server/threads/manageThreads.js +526 -63
  500. package/server/threads/socketRouter.ts +81 -26
  501. package/server/threads/threadHeapMemory.ts +26 -0
  502. package/server/threads/threadServer.js +43 -15
  503. package/sqlTranslator/index.ts +31 -8
  504. package/studio/web/assets/{Chat-Br06zdMA.js → Chat-BnCBegQz.js} +1 -1
  505. package/studio/web/assets/{FloatingChat-BWImX5fA.js → FloatingChat-CoDW1ySS.js} +3 -3
  506. package/studio/web/assets/{apiToken-DN0nmDsq.js → apiToken-Bwk5BLXW.js} +1 -1
  507. package/studio/web/assets/{applications-kSxVoyeU.js → applications-DHxGi7JH.js} +1 -1
  508. package/studio/web/assets/{cssMode-C1JeufH5.js → cssMode-s0cWI-_M.js} +1 -1
  509. package/studio/web/assets/{editor-qoo9CrGO.js → editor-DNcRHK54.js} +1 -1
  510. package/studio/web/assets/{html-Dt4IIy04.js → html-Bdssedlg.js} +1 -1
  511. package/studio/web/assets/{htmlMode-DXgKKr4C.js → htmlMode-CoDlJ3fw.js} +1 -1
  512. package/studio/web/assets/{index-6onkYFOG.js → index-D6sxmFLR.js} +5 -5
  513. package/studio/web/assets/{index.lazy-BrCFnpNJ.js → index.lazy-tmU5BS8s.js} +1 -1
  514. package/studio/web/assets/{javascript-DNCQGUBc.js → javascript-B8meVSTH.js} +1 -1
  515. package/studio/web/assets/{jsonMode-CR6HWruP.js → jsonMode-DpIPd35T.js} +1 -1
  516. package/studio/web/assets/{languageServices-BM4fI4rS.js → languageServices-C_5FMJzQ.js} +1 -1
  517. package/studio/web/assets/{lspLanguageFeatures-DSa1ttcD.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
  518. package/studio/web/assets/{notifications-BHXLnh6x.js → notifications-CQf18QKb.js} +1 -1
  519. package/studio/web/assets/{notifications-CUmtIA6z.js → notifications-CvZivSbh.js} +1 -1
  520. package/studio/web/assets/{profile-8BeFSF3j.js → profile-DdOwtntb.js} +1 -1
  521. package/studio/web/assets/{regions-C8qR0HhD.js → regions-n69fwagr.js} +1 -1
  522. package/studio/web/assets/{register-B4n5i0SD.js → register-PfWTCXWB.js} +2 -2
  523. package/studio/web/assets/{setComponentFile-g0_B0lgX.js → setComponentFile-Bg6O7X0S.js} +1 -1
  524. package/studio/web/assets/{setup-B0CTj_Q5.js → setup-CUx_aUDl.js} +2 -2
  525. package/studio/web/assets/{status-C6Yib7-K.js → status-D7BVKqX9.js} +1 -1
  526. package/studio/web/assets/{toggleHighContrast-Dgta7bVi.js → toggleHighContrast-DBSyXzMr.js} +1 -1
  527. package/studio/web/assets/{tsMode-CH_jHvU-.js → tsMode-BByKCjBS.js} +1 -1
  528. package/studio/web/assets/{typescript-Co9LCXd5.js → typescript-DDLnLpw9.js} +1 -1
  529. package/studio/web/assets/{useEntityRestURL-wKC8NsC_.js → useEntityRestURL-31CHGaHk.js} +1 -1
  530. package/studio/web/assets/{workers-CWeLxCXA.js → workers-pR3jRY9D.js} +1 -1
  531. package/studio/web/assets/{xml-BadC-0Rk.js → xml-2iRnMhQO.js} +1 -1
  532. package/studio/web/assets/{yaml-BiUfxPbC.js → yaml-Bf92gJpd.js} +1 -1
  533. package/studio/web/index.html +1 -1
  534. package/upgrade/directives/5-3-0.ts +132 -0
  535. package/upgrade/directives/directivesController.ts +2 -1
  536. package/utility/OperationFunctionCaller.ts +2 -1
  537. package/utility/common_utils.ts +30 -5
  538. package/utility/componentNames.ts +12 -0
  539. package/utility/environment/environmentManager.ts +3 -7
  540. package/utility/environment/systemInformation.ts +7 -0
  541. package/utility/errors/commonErrors.ts +4 -0
  542. package/utility/errors/hdbError.ts +29 -0
  543. package/utility/hdbTerms.ts +17 -0
  544. package/utility/logging/harper_logger.ts +87 -14
  545. package/utility/logging/logRotator.ts +72 -45
  546. package/utility/nodeIdentity.ts +45 -0
  547. package/utility/npmUtilities.ts +10 -8
  548. package/utility/operationPermissions.ts +18 -1
  549. package/utility/operation_authorization.ts +231 -42
  550. package/utility/watchPath.ts +63 -0
  551. package/utility/watcherFallback.ts +270 -0
  552. package/validation/configValidator.ts +209 -72
  553. package/validation/installValidator.ts +15 -0
  554. package/validation/validationWrapper.ts +18 -4
@@ -1,7 +1,8 @@
1
1
  import { cleanupUnusedBlobs, collectRetainedFileIds } from './blob.ts';
2
2
  import { Transaction as LMDBTransaction } from 'lmdb';
3
3
  import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
4
- import { ServerError } from '../utility/errors/hdbError.ts';
4
+ import { ServerError, TransactionCommitConflictTimeoutError } from '../utility/errors/hdbError.ts';
5
+ import { lockNotHeldError, type RecordLockHandle } from './recordLock.ts';
5
6
  import * as harperLogger from '../utility/logging/harper_logger.ts';
6
7
  import type { Context, Id } from './ResourceInterface.ts';
7
8
  import * as envMngr from '../utility/environment/environmentManager.ts';
@@ -14,9 +15,17 @@ const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE;
14
15
  import type { RootDatabaseKind } from './databases.ts';
15
16
  import type { Entry } from './RecordEncoder.ts';
16
17
  import { toBufferKey } from 'ordered-binary';
18
+ import {
19
+ createLongLivedHolderReportBudget,
20
+ describeHolderCandidates,
21
+ finishLongLivedHolderReports,
22
+ getReportThresholdMs,
23
+ rebaseLongLivedTransactionReports,
24
+ reportLongLivedHolder,
25
+ type LongLivedHolderReportBudget,
26
+ } from './longLivedTransactions.ts';
17
27
 
18
28
  const trackedTxns = new Set<DatabaseTransaction>();
19
- const readTransactionOwners = new WeakMap<ReadTransaction, DatabaseTransaction>();
20
29
  // Read options for a rotated generation's native transactions; shared because they never vary.
21
30
  const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
22
31
  // Logical transactions the monitor supervises for their WRITES, kept apart from trackedTxns because the
@@ -24,7 +33,7 @@ const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
24
33
  // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
25
34
  // chain root — so a chain child can never become its own timeout root (issue #2231).
26
35
  const supervisedWriteRoots = new Set<DatabaseTransaction>();
27
- const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
36
+ let MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
28
37
  const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS);
29
38
  export const TRANSACTION_STATE = {
30
39
  CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads)
@@ -36,6 +45,12 @@ export const TRANSACTION_STATE = {
36
45
  LINGERING: 2,
37
46
  };
38
47
  const MAX_RETRIES = 40;
48
+ // Over-limit monitor ticks a transaction parked in its commit phase is spared before it is aborted
49
+ // like any other over-time transaction. Sparing re-arms `timeout`, so each of these costs two
50
+ // monitor intervals — roughly 10 minutes at the 30s default. Generous, so a legitimately large blob
51
+ // write finishes, but bounded, so a pre-commit source that stalls instead of finishing can't pin its
52
+ // read snapshot forever (issue #2062).
53
+ export const COMMIT_PHASE_GRACE = 10;
39
54
  // Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
40
55
  // cap (see the commit rejection handler), don't grow the delay unbounded.
41
56
  const MAX_RETRY_DELAY_MS = 1000;
@@ -69,7 +84,33 @@ let outstandingCommitCount = 0;
69
84
  // the whole thread, regardless of how many distinct commits individually cross the threshold — see
70
85
  // the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
71
86
  const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
72
- let lastOverloadLogAt = -Infinity;
87
+ // One cooldown per reporting site, not one shared: `shed` fires on every bystander write during a
88
+ // wedge and would starve `abandon`, which names a different transaction and is the only server-side
89
+ // record of why a request was failed. Mutates only when it grants, so a caller with its own
90
+ // suppression as well (checkOverloaded's per-node `logged`) must test that first.
91
+ const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
92
+ function allowStuckCommitLog(site: 'shed' | 'abandon', now: number): boolean {
93
+ if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS) return false;
94
+ lastStuckCommitLogAt[site] = now;
95
+ return true;
96
+ }
97
+
98
+ // Which database/table, which native transaction, and which request a stuck commit belongs to —
99
+ // without it a wedge gives no indication of what to investigate (harper#2001).
100
+ function describeCommitIdentity(
101
+ store: any,
102
+ startedFrom: { resourceName: string; method: string } | undefined,
103
+ nativeTransaction: any
104
+ ): string {
105
+ const nativeTransactionId = nativeTransaction?.id;
106
+ return (
107
+ `from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
108
+ (nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
109
+ (startedFrom?.resourceName
110
+ ? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
111
+ : '')
112
+ );
113
+ }
73
114
 
74
115
  // Track a submitted commit until it settles. Every attempt is tracked unconditionally: a
75
116
  // coordinated retry round and a chained second-store commit are both issued from inside the
@@ -146,6 +187,20 @@ export function getOutstandingCommits(): { count: number; oldestAgeMs: number |
146
187
  // about the caller's pattern, not the individual commit.
147
188
  let replayedWritesWarned = false;
148
189
 
190
+ /**
191
+ * Abort a detached native handle. RocksTransaction.abort() throws on one that was already
192
+ * committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
193
+ * throw there would abandon the rest of the cleanup.
194
+ */
195
+ function abortNativeTransaction(transaction: RocksTransaction | null | undefined, context: string): void {
196
+ if (transaction == null) return;
197
+ try {
198
+ transaction.abort();
199
+ } catch (error) {
200
+ harperLogger.debug?.(context, error);
201
+ }
202
+ }
203
+
149
204
  // The analytics module registers a recorder here at load (dependency inversion, mirroring
150
205
  // `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
151
206
  // modules. Unset until analytics loads, and when analytics is disabled the recorder call is cheap.
@@ -237,76 +292,6 @@ export function transactionOpenTooLongError(): ServerError {
237
292
  );
238
293
  }
239
294
 
240
- class ReadSnapshotExpiredError extends ServerError {
241
- constructor() {
242
- super('Read scan snapshot expired; retry the read without replaying previously committed writes', 503);
243
- this.name = 'ReadSnapshotExpiredError';
244
- }
245
- }
246
-
247
- export function trackReadRange(transaction: ReadTransaction, createRange: () => any): any {
248
- const owner = readTransactionOwners.get(transaction);
249
- if (!owner) return createRange();
250
- function checkActive() {
251
- if (owner.timedOut) throw transactionOpenTooLongError();
252
- if (owner.transaction !== transaction) {
253
- throw new ReadSnapshotExpiredError();
254
- }
255
- }
256
- checkActive();
257
- const range = createRange();
258
- const iterate = range.iterate;
259
- range.iterate = function (options) {
260
- const iterator = iterate.call(this, options);
261
- let done = false;
262
- // Closing the underlying iterator is the one step here that can throw for a reason the caller
263
- // must not see: `next()` only reaches it once the snapshot is already gone, which is the
264
- // likeliest moment for the native layer to object, and an error from cleanup would replace the
265
- // named 503 with exactly the raw iterator error this wrapper exists to stop surfacing. The
266
- // closure reference rather than `this` so a destructured `next` still cleans up.
267
- const wrapper = {
268
- [Symbol.iterator]() {
269
- return this;
270
- },
271
- next() {
272
- if (done) return { done: true, value: undefined };
273
- try {
274
- checkActive();
275
- owner.rangeReadActive = true;
276
- const result = iterator.next();
277
- done = result.done === true;
278
- return result;
279
- } catch (error) {
280
- closeQuietly();
281
- throw error;
282
- }
283
- },
284
- return(value?: any) {
285
- if (!done) {
286
- done = true;
287
- iterator.return?.(value);
288
- }
289
- return { done: true, value };
290
- },
291
- throw(error) {
292
- // Not delegated to `iterator.throw`: it closes and rethrows the same error anyway, and
293
- // delegating after the close below would run it against an iterator already closed.
294
- closeQuietly();
295
- throw error;
296
- },
297
- };
298
- function closeQuietly() {
299
- try {
300
- wrapper.return();
301
- } catch {
302
- // the error being propagated is the actionable one
303
- }
304
- }
305
- return wrapper;
306
- };
307
- return range;
308
- }
309
-
310
295
  type MaybePromise<T> = T | Promise<T>;
311
296
 
312
297
  export type CommitOptions = {
@@ -336,6 +321,9 @@ export type TransactionWrite = {
336
321
  // overload accounting, the replay marker and a no-op write's removal all belong to the committer.
337
322
  validate?: (txnTime: number, committedBy: DatabaseTransaction) => void;
338
323
  fullUpdate?: boolean;
324
+ // The origin record version carried by an applied or replayed write. Bound it by the origin's
325
+ // transaction-log key so malformed or historical overloaded values cannot move ordering past the write.
326
+ recordVersion?: number;
339
327
  saved?: boolean;
340
328
  deferSave?: boolean;
341
329
  skipReplicationConfirmation?: boolean;
@@ -374,8 +362,31 @@ export type TransactionWrite = {
374
362
  // this write appended an audit entry, which references its saved blobs — they then belong to the
375
363
  // audit trail (audit pruning deletes them), so the superseded-write cleanup must leave them alone
376
364
  blobsAuditReferenced?: boolean;
365
+ // Lock handle set by Table._writeUpdate when the write is staged through a held record lock;
366
+ // used in DatabaseTransaction.save() to assign and track that handle's write versions.
367
+ lockHandle?: RecordLockHandle;
368
+ // Per-operation holder version, set once on first save() and reused on retry so that
369
+ // ImmediateTransaction's sequential immediateCommit saves don't collide: each write carries its
370
+ // own stamp independently of this.timestamp, which may not reset to 0 between saves.
371
+ lockStamp?: number;
372
+ // Version staged by a write that actually changed the record this round. It advances any
373
+ // same-key held handle's floor only after the native transaction commits successfully.
374
+ appliedRecordVersion?: number;
375
+ // Present only while a transaction owns record locks, keeping bookkeeping off ordinary writes.
376
+ trackRecordVersion?: boolean;
377
+ // Set by a table commit handler only when this retry round staged a record change.
378
+ recordVersionApplied?: boolean;
379
+ // Set by DatabaseTransaction.save() on the immediateCommit path: the Promise returned by the
380
+ // inner this.commit({ transaction }) call. The ImmediateTransaction outer commit resolves before
381
+ // this settles (fire-and-forget from the if-branch), so Table.save()'s lock-writable path awaits
382
+ // it to ensure the write is durable before resolving to the caller.
383
+ innerCommit?: MaybePromise<CommitResolution>;
377
384
  };
378
385
 
386
+ export function getAppliedWriteVersion(recordVersion: number | undefined, txnLogKey: number): number {
387
+ return recordVersion == null ? txnLogKey : Math.min(recordVersion, txnLogKey);
388
+ }
389
+
379
390
  /**
380
391
  * The state a preceding write in this transaction left for `operation`'s key, or undefined if this
381
392
  * is the first write to it. Within a transaction the writes are ordered by program order, and
@@ -435,7 +446,11 @@ export class DatabaseTransaction implements Transaction {
435
446
  // link with no writes that is being read in a loop would otherwise masquerade as write activity and
436
447
  // keep a write-holding head immortal.
437
448
  declare writeTimeout: number;
449
+ writeTick = -1;
438
450
  timeoutBudget = 0;
451
+ // Initialized rather than `declare`d so the class shape stays monomorphic on the write path. One clock
452
+ // read per handle is what lets the monitor age a handle without calling into the registry every tick.
453
+ handleOpenedAt = 0;
439
454
  // save() only stages here; ImmediateTransaction overrides it to commit, which addWrite must not defer
440
455
  saveCommits = false;
441
456
  // True where save() puts the write into this transaction's native handle, which is what lets a scope
@@ -449,6 +464,13 @@ export class DatabaseTransaction implements Transaction {
449
464
  declare next: DatabaseTransaction;
450
465
  // The head of this multi-store chain, set when the link is created; absent on the head itself.
451
466
  declare root?: DatabaseTransaction;
467
+ // When this logical commit first reached the storage engine, held on the chain root so every
468
+ // retry round and every chained store measures ONE elapsed wait (issue #2450). Deliberately not
469
+ // the per-attempt clock trackOutstandingCommit() keeps: that one drives thread-wide load
470
+ // shedding and must stay per-attempt, or a long uncapped source-apply retry would 503 every
471
+ // unrelated request on the thread. Cleared when the logical commit settles, so a reused
472
+ // transaction's next batch starts on a fresh budget.
473
+ declare commitStartedAt?: number;
452
474
  // Whether this link is why its chain root is write-supervised (see endWriteSupervision).
453
475
  declare writeSupervised?: boolean;
454
476
  declare stale: boolean;
@@ -489,10 +511,31 @@ export class DatabaseTransaction implements Transaction {
489
511
  // asks to stop reading a pinned snapshot, so re-pinning one for the rest of the scope would take
490
512
  // back what it asked for.
491
513
  snapshotFree = false;
514
+ // Set while commit() is parked in its pre-commit await (`before`/`beforeIntermediate` completions —
515
+ // in practice a blob's durable file write). The write set is sealed and the caller is awaiting the
516
+ // commit, so this is core's own I/O rather than an application holding a transaction open, which is
517
+ // what the open-transaction limit polices: the monitor spares it for COMMIT_PHASE_GRACE ticks
518
+ // instead of poisoning it (issue #2062).
519
+ committing = false;
520
+ commitPhaseTicks = 0;
521
+ declare commitChainHead?: DatabaseTransaction;
522
+ // O(1) lookup in recordLockFor; only lock() handles are registered here (no gate handles).
523
+ declare recordLocks?: Map<any, Map<unknown, RecordLockHandle>>;
524
+ // Tracks in-flight acquireRecordKey calls so concurrent lock() calls for the same key in one
525
+ // link (e.g. Promise.all([T.lock(id), T.lock(id)])) can coalesce rather than self-block.
526
+ declare pendingLocks?: Map<any, Map<unknown, Promise<RecordLockHandle>>>;
492
527
 
493
- rangeReadActive = false;
528
+ setCommitPhase(committing: boolean): void {
529
+ // A commit phase covers the sealed write set across the whole multi-store chain.
530
+ for (let txn: DatabaseTransaction = this; txn; txn = txn.next) {
531
+ txn.committing = committing;
532
+ txn.commitChainHead = committing ? this : undefined;
533
+ if (committing) txn.commitPhaseTicks = 0;
534
+ }
535
+ }
494
536
 
495
- renewReadTimeout(): void {
537
+ getReadTxn(disableSnapshot?: boolean): ReadTransaction {
538
+ this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
496
539
  // The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
497
540
  // while no uncommitted writes are held: staged writes hold write intents that other writers'
498
541
  // coordinated-retry commits park on, so a handler that wrote once and then only reads — an
@@ -505,11 +548,6 @@ export class DatabaseTransaction implements Transaction {
505
548
  if ((this.writes.length === 0 && !this.next) || this.open !== TRANSACTION_STATE.OPEN || !this.hasPendingWrites()) {
506
549
  this.timeout = Math.max(txnExpiration, this.timeoutBudget);
507
550
  }
508
- }
509
-
510
- getReadTxn(disableSnapshot?: boolean): ReadTransaction {
511
- this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
512
- this.renewReadTimeout();
513
551
  if (this.transaction) {
514
552
  if ((this.transaction as any).openTimer) (this.transaction as any).openTimer = 0;
515
553
  return this.transaction;
@@ -544,10 +582,9 @@ export class DatabaseTransaction implements Transaction {
544
582
  // Monitor state is not ownership state: it stays with `trackedTxns.add` in getReadTxn().
545
583
  private attachOwnedTransaction(transaction: RocksTransactionWithRetry): void {
546
584
  this.transaction = transaction;
547
- readTransactionOwners.set(transaction, this);
548
- this.rangeReadActive = false;
549
585
  this.readTxnsUsed = 1;
550
586
  this.baseReadRefConsumed = false;
587
+ this.handleOpenedAt = performance.now();
551
588
  }
552
589
 
553
590
  /**
@@ -586,9 +623,9 @@ export class DatabaseTransaction implements Transaction {
586
623
  trackedTxns.delete(this);
587
624
  this.endWriteSupervision();
588
625
  this.transaction = null;
589
- this.rangeReadActive = false;
590
626
  this.readTxnsUsed = 0;
591
627
  this.readTxnRefCount = 0;
628
+ this.handleOpenedAt = 0;
592
629
  return transaction;
593
630
  }
594
631
 
@@ -635,11 +672,7 @@ export class DatabaseTransaction implements Transaction {
635
672
  */
636
673
  releaseReadTxn(): void {
637
674
  const transaction = this.detachOwnedTransaction();
638
- try {
639
- transaction?.abort();
640
- } catch (error) {
641
- harperLogger.debug?.('releasing timed-out read transaction', error);
642
- }
675
+ abortNativeTransaction(transaction, 'releasing timed-out read transaction');
643
676
  this.completeDeferredContextRelease();
644
677
  }
645
678
 
@@ -712,6 +745,70 @@ export class DatabaseTransaction implements Transaction {
712
745
  }
713
746
  }
714
747
 
748
+ registerRecordLock(handle: RecordLockHandle): void {
749
+ if (!this.recordLocks) this.recordLocks = new Map();
750
+ let storeMap = this.recordLocks.get(handle.store);
751
+ if (!storeMap) this.recordLocks.set(handle.store, (storeMap = new Map()));
752
+ storeMap.set(handle.keyId, handle);
753
+ }
754
+
755
+ recordLockFor(store: any, keyId: unknown): RecordLockHandle | undefined {
756
+ const storeMap = this.recordLocks?.get(store);
757
+ if (!storeMap) return undefined;
758
+ const h = storeMap.get(keyId);
759
+ if (!h) return undefined;
760
+ if (!h.released) return h;
761
+ // Prune released handles so they don't accumulate; an expired handle checking re-entrancy
762
+ // would otherwise be seen as the holder and incorrectly granted access.
763
+ storeMap.delete(keyId);
764
+ return undefined;
765
+ }
766
+
767
+ unregisterRecordLock(handle: RecordLockHandle): void {
768
+ const storeMap = this.recordLocks?.get(handle.store);
769
+ if (storeMap?.get(handle.keyId) === handle) storeMap.delete(handle.keyId);
770
+ }
771
+
772
+ registerPendingLock(store: any, keyId: unknown, pending: Promise<RecordLockHandle>): void {
773
+ if (!this.pendingLocks) this.pendingLocks = new Map();
774
+ let storeMap = this.pendingLocks.get(store);
775
+ if (!storeMap) this.pendingLocks.set(store, (storeMap = new Map()));
776
+ storeMap.set(keyId, pending);
777
+ }
778
+
779
+ pendingLockFor(store: any, keyId: unknown): Promise<RecordLockHandle> | undefined {
780
+ return this.pendingLocks?.get(store)?.get(keyId);
781
+ }
782
+
783
+ unregisterPendingLock(store: any, keyId: unknown): void {
784
+ this.pendingLocks?.get(store)?.delete(keyId);
785
+ }
786
+
787
+ /** Release every transaction-scoped lock() handle this link owns. */
788
+ releaseRecordLocks(): void {
789
+ const recordLocks = this.recordLocks;
790
+ if (!recordLocks) return;
791
+ for (const [store, storeMap] of recordLocks) {
792
+ for (const [keyId, handle] of storeMap) {
793
+ if (handle.hold) continue; // hold handles outlive the transaction; released by unlock()
794
+ handle.release();
795
+ storeMap.delete(keyId);
796
+ }
797
+ if (storeMap.size === 0) recordLocks.delete(store);
798
+ }
799
+ if (recordLocks.size === 0) this.recordLocks = undefined;
800
+ }
801
+
802
+ private noteCommittedLockVersions(): void {
803
+ const recordLocks = this.recordLocks;
804
+ if (!recordLocks) return;
805
+ for (const write of this.writes) {
806
+ if (write?.appliedRecordVersion == null) continue;
807
+ const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
808
+ if (handle && !handle.released) handle.noteHolderVersion(write.appliedRecordVersion);
809
+ }
810
+ }
811
+
715
812
  /**
716
813
  * Discard the staged write set (committed or aborted); the per-key chain must go with it so a
717
814
  * reused transaction never bases a write on a previous batch's staged state.
@@ -771,26 +868,28 @@ export class DatabaseTransaction implements Transaction {
771
868
  // original per-request log was fixed to avoid, just shifted from per-request to per-commit.
772
869
  // A commit skipped by the cooldown is NOT marked `logged`, so it still gets a log later if
773
870
  // it's still the oldest once the cooldown clears, rather than going silent forever.
774
- if (!oldestOutstandingCommit.logged && now - lastOverloadLogAt > OVERLOAD_LOG_MIN_INTERVAL_MS) {
871
+ if (!oldestOutstandingCommit.logged && allowStuckCommitLog('shed', now)) {
775
872
  // Log once per stuck commit (not once per rejected request, harper#2001): a wedged
776
873
  // thread otherwise logs nothing at all server-side while rejecting every write with a
777
874
  // 503, which was the single biggest obstacle to root-causing a recurrence. The flag lives
778
875
  // on the node itself, so if THIS commit settles while still over the limit and a
779
876
  // different one is now oldest, that one logs too instead of staying silent forever.
780
877
  oldestOutstandingCommit.logged = true;
781
- lastOverloadLogAt = now;
782
- const nativeTransactionId = oldestOutstandingCommit.nativeTransaction?.id;
783
- const store = oldestOutstandingCommit.store;
784
- const startedFrom = oldestOutstandingCommit.startedFrom;
785
878
  harperLogger.error(
786
879
  `Rejecting writes on this thread: a commit has been outstanding for ` +
787
880
  `${Math.round(now - oldestOutstandingCommit.start)}ms (exceeds the ` +
788
- `${MAX_OUTSTANDING_TXN_DURATION}ms limit), from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
789
- (nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
790
- (startedFrom?.resourceName
791
- ? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
792
- : '') +
793
- `. Further record updates and publishes from new application requests on this thread ` +
881
+ `${MAX_OUTSTANDING_TXN_DURATION}ms limit), ` +
882
+ describeCommitIdentity(
883
+ oldestOutstandingCommit.store,
884
+ oldestOutstandingCommit.startedFrom,
885
+ oldestOutstandingCommit.nativeTransaction
886
+ ) +
887
+ `.` +
888
+ describeHolderCandidates(
889
+ oldestOutstandingCommit.store?.rootStore?.path,
890
+ oldestOutstandingCommit.nativeTransaction?.id
891
+ ) +
892
+ ` Further record updates and publishes from new application requests on this thread ` +
794
893
  `will be rejected with 503 until the commit settles or the process is restarted (deletes, ` +
795
894
  `and writes applied from a canonical source, e.g. replication or a caching source, bypass this check).`
796
895
  );
@@ -854,6 +953,7 @@ export class DatabaseTransaction implements Transaction {
854
953
  // Independent write-recency signal for chainStillActive (see the field comment) — reads never
855
954
  // touch this, only writes do.
856
955
  this.writeTimeout = this.timeout;
956
+ this.writeTick = monitorTick;
857
957
  this.linkWrite(operation);
858
958
  this.writes.push(operation);
859
959
  operation.stagedIn = this;
@@ -884,7 +984,32 @@ export class DatabaseTransaction implements Transaction {
884
984
  }
885
985
 
886
986
  save(operation: TransactionWrite, transaction?: RocksTransaction, reloadEntry = false, options?: CommitOptions) {
887
- let txnTime = this.timestamp;
987
+ const lockHandle = operation.lockHandle;
988
+ // Guard: a write staged through an expired or released lock handle must not land.
989
+ // The handle's lease timer already unlocked the native key; another holder may have taken it.
990
+ if (lockHandle && (lockHandle.expired || lockHandle.released)) {
991
+ // Remove the operation from the staged set so subsequent writes on this context do not
992
+ // re-throw 409 due to a stale null-saved entry sitting in this.writes.
993
+ const failedIdx = this.writes.indexOf(operation);
994
+ if (failedIdx > -1) this.writes[failedIdx] = null;
995
+ throw lockNotHeldError(lockHandle);
996
+ }
997
+ // Lock-write timestamp rules.
998
+ if (lockHandle) {
999
+ if (this.open === TRANSACTION_STATE.CLOSED || this.saveCommits) {
1000
+ // CLOSED path (second+ write per ImmediateTransaction cycle) OR the first write in
1001
+ // an ImmediateTransaction (open=OPEN until commit sets it CLOSED, but saveCommits
1002
+ // signals the per-write-commit semantics): stamp with nextHolderVersion() so that
1003
+ // each sequential save() gets its own monotonically increasing stamp — scoped or hold.
1004
+ // The stamp stays on the operation and is consumed below rather than assigned to
1005
+ // this.timestamp: pinning the link's clock would stamp every OTHER write staged on
1006
+ // the same context before the commit resets it — a concurrent write in the caller's
1007
+ // own Promise.all, or the next operation in a retry/replay save loop — with the
1008
+ // lock's version, which LWW then silently drops against a newer record version.
1009
+ if (!operation.lockStamp) operation.lockStamp = lockHandle.holderVersionCandidate();
1010
+ }
1011
+ }
1012
+ let txnTime = operation.lockStamp ?? this.timestamp;
888
1013
  // Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
889
1014
  // be retained for outstanding read iterators; staging into it would silently discard the write
890
1015
  // when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
@@ -929,13 +1054,19 @@ export class DatabaseTransaction implements Transaction {
929
1054
  (transaction as RocksTransactionWithRetry).isRetry = true;
930
1055
  }
931
1056
  if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
1057
+ // `txnTime` is this transaction's timestamp — the key its entries take in the per-origin log.
1058
+ // A write applied from elsewhere carries the origin's record version too, and that is what the
1059
+ // record is stored at; the two coincide for every locally-originated write. Gated on the apply
1060
+ // flags so an ordinary write never reads the property (harper#2412).
1061
+ const writeVersion =
1062
+ this.sourceApply || this.isReplay ? getAppliedWriteVersion(operation.recordVersion, txnTime) : txnTime;
932
1063
  if (reloadEntry || operation.entry === undefined) {
933
1064
  operation.entry = operation.store.getEntry(operation.key, { transaction });
934
1065
  }
935
1066
  if (!operation.saved) {
936
1067
  operation.saved = true;
937
1068
  // immediately execute in this transaction
938
- if ((operation.validate?.(txnTime, this) as any) === false) {
1069
+ if ((operation.validate?.(writeVersion, this) as any) === false) {
939
1070
  operation.commit = () => {}; // noop if we try again
940
1071
  return;
941
1072
  }
@@ -944,7 +1075,11 @@ export class DatabaseTransaction implements Transaction {
944
1075
  result = operation.beforeIntermediate?.() as Promise<void>;
945
1076
  if (result?.then) this.stageCompletion(result);
946
1077
  }
947
- const completion = operation.commit(txnTime, operation.entry, this.retries > 0, transaction) as Promise<void>;
1078
+ if (lockHandle || this.recordLocks) operation.trackRecordVersion = true;
1079
+ if (operation.trackRecordVersion) operation.recordVersionApplied = false;
1080
+ const completion = operation.commit(writeVersion, operation.entry, this.retries > 0, transaction) as Promise<void>;
1081
+ if (operation.trackRecordVersion)
1082
+ operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
948
1083
  if (typeof completion?.then === 'function') this.stageCompletion(completion);
949
1084
  // Sticky record that THIS write staged with its audit entry appended (log entries batch on the
950
1085
  // native transaction and are durably written by its commit attempt — even a failed one — so
@@ -956,7 +1091,13 @@ export class DatabaseTransaction implements Transaction {
956
1091
  operation.appendedAuditEntry = true;
957
1092
  }
958
1093
  if (immediateCommit) {
959
- return this.commit({ ...options, transaction }); // immediately commit if the harper transaction is closed
1094
+ // immediately commit if the harper transaction is closed
1095
+ const innerCommit = this.commit({ ...options, transaction });
1096
+ // Expose on the operation so the lock-writable Table.save() path can await the real
1097
+ // native commit — without this the ImmediateTransaction outer commit resolves before
1098
+ // the native transaction.commit() settles (fire-and-forget from the if-branch).
1099
+ operation.innerCommit = innerCommit;
1100
+ return innerCommit;
960
1101
  }
961
1102
  }
962
1103
 
@@ -968,17 +1109,35 @@ export class DatabaseTransaction implements Transaction {
968
1109
  // reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
969
1110
  // but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
970
1111
  let transaction = options.transaction ?? this.transaction;
971
- for (let i = 0; i < this.writes.length; i++) {
972
- let operation = this.writes[i];
973
- if (!operation || (this.retries === 0 && operation.saved)) continue;
974
- this.save(operation, transaction, i < this.validated, options);
1112
+ try {
1113
+ for (let i = 0; i < this.writes.length; i++) {
1114
+ let operation = this.writes[i];
1115
+ if (!operation || (this.retries === 0 && operation.saved)) continue;
1116
+ this.save(operation, transaction, i < this.validated, options);
1117
+ }
1118
+ } catch (error) {
1119
+ this.abort();
1120
+ throw error;
975
1121
  }
976
1122
  this.validated = this.writes.length;
977
1123
  const completions = this.completions;
978
1124
  if (completions.length > 0) this.completions = []; // reset
1125
+ const stagedWrites = this.writes.length;
1126
+ if (completions.length > 0) {
1127
+ this.setCommitPhase(true);
1128
+ }
979
1129
  return when(
980
1130
  completions.length > 0 ? Promise.all(completions) : null,
981
1131
  () => {
1132
+ if (completions.length > 0) this.setCommitPhase(false);
1133
+ // The transaction can be aborted underneath us while we are parked in the await above — by the
1134
+ // monitor once the commit phase outlives its grace, or through the multi-store poison chain.
1135
+ // abort() cleared the write set and released the handle, so resuming would commit nothing and
1136
+ // resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
1137
+ // carrying a blob is left holding an instance whose file was unlinked (issue #2062).
1138
+ if (this.timedOut) throw transactionOpenTooLongError();
1139
+ if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
1140
+ throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
982
1141
  if (this.writes.length > this.validated) {
983
1142
  // check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
984
1143
  return this.commit(options);
@@ -1075,8 +1234,6 @@ export class DatabaseTransaction implements Transaction {
1075
1234
  if (transaction) {
1076
1235
  this.writes = this.writes.filter((write) => write); // filter out removed entries
1077
1236
  if (this.writes.length > 0) {
1078
- // Commit retries can construct fresh ranges on this live handle after read ownership ends.
1079
- readTransactionOwners.delete(transaction);
1080
1237
  // The transaction was created with coordinatedRetry:true (see
1081
1238
  // getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
1082
1239
  // sentinel (a number) is why commitResolution is typed
@@ -1114,8 +1271,12 @@ export class DatabaseTransaction implements Transaction {
1114
1271
  // claimed this per-database transaction in txnForContext and so can name the wrong
1115
1272
  // table when a transaction spans more than one table in the same database.
1116
1273
  trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
1274
+ // Every retry round and every chained store re-enters here and must inherit the chain
1275
+ // root's clock rather than restart it, so only the first submission stamps.
1276
+ const chainRoot = this.root ?? this;
1277
+ if (chainRoot.commitStartedAt == null) chainRoot.commitStartedAt = performance.now();
1117
1278
  const completions = [];
1118
- return commitResolution.then(
1279
+ const commitOutcome = commitResolution.then(
1119
1280
  (commitResult) => {
1120
1281
  if (commitResult === RETRY_NOW_VALUE) {
1121
1282
  this.retries++;
@@ -1123,6 +1284,8 @@ export class DatabaseTransaction implements Transaction {
1123
1284
  // Mark this specific native transaction as a retry so RocksTransactionLogStore
1124
1285
  // skips re-writing its already-staged txn-log entries (#2).
1125
1286
  (transaction as RocksTransactionWithRetry).isRetry = true;
1287
+ const pastBudget = this.elapsedPastCommitBudget();
1288
+ if (pastBudget) this.abandonCommitAfterDeadline(transaction, pastBudget);
1126
1289
  // Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
1127
1290
  // at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
1128
1291
  if (this.retries > MAX_RETRIES) {
@@ -1182,9 +1345,11 @@ export class DatabaseTransaction implements Transaction {
1182
1345
  if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
1183
1346
  cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
1184
1347
  }
1348
+ if (this.recordLocks) this.noteCommittedLockVersions();
1185
1349
  // now reset transactions tracking; this transaction be reused and committed again
1186
1350
  this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
1187
1351
  this.clearWrites();
1352
+ this.releaseRecordLocks();
1188
1353
  if (options.doneWriting) this.endScopeOwnership();
1189
1354
  this.releaseContext(!!options.doneWriting);
1190
1355
  let txnTime = this.timestamp;
@@ -1239,6 +1404,10 @@ export class DatabaseTransaction implements Transaction {
1239
1404
  // premature publish, and no fresh-transaction replay that would drop the entry.
1240
1405
  // Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
1241
1406
  (transaction as RocksTransactionWithRetry).isRetry = true;
1407
+ // Before the backoff gate below: the budget can already be spent on the first
1408
+ // retry when an earlier store in this chain consumed it.
1409
+ const pastBudget = this.elapsedPastCommitBudget();
1410
+ if (pastBudget) this.abandonCommitAfterDeadline(transaction, pastBudget);
1242
1411
  if (this.retries > 2) {
1243
1412
  // Transactions applying data from a canonical source of truth (replication peer or
1244
1413
  // external caching source) must never drop a write on a transient conflict: there is no
@@ -1287,24 +1456,45 @@ export class DatabaseTransaction implements Transaction {
1287
1456
  // finished and its durability is unknown, so ownership goes with it.
1288
1457
  this.endScopeOwnership();
1289
1458
  this.releaseContext(!!options.doneWriting);
1459
+ this.releaseRecordLocks();
1460
+ this.timestamp = 0;
1290
1461
  throw error;
1291
1462
  }
1292
1463
  }
1293
1464
  );
1465
+ // `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
1466
+ // chained stores' own commits — so releasing here covers every terminal exit (success,
1467
+ // retry exhaustion, abandonment, terminal failure) in one place rather than five.
1468
+ // Released through the RETURNED promise rather than a second subscriber on
1469
+ // `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
1470
+ // silence the unhandled-rejection that surfaces it.
1471
+ return commitOutcome.then(
1472
+ (resolution) => {
1473
+ chainRoot.commitStartedAt = undefined;
1474
+ return resolution;
1475
+ },
1476
+ (error) => {
1477
+ chainRoot.commitStartedAt = undefined;
1478
+ throw error;
1479
+ }
1480
+ );
1294
1481
  }
1295
1482
  for (const write of this.writes) {
1296
1483
  if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
1297
1484
  cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
1298
1485
  }
1486
+ if (this.recordLocks) this.noteCommittedLockVersions();
1299
1487
  this.clearWrites();
1488
+ this.releaseRecordLocks();
1300
1489
  if (options.doneWriting) this.endScopeOwnership();
1301
1490
  this.releaseContext(!!options.doneWriting);
1302
1491
  const txnResolution: CommitResolution = {
1303
1492
  txnTime: this.timestamp,
1304
1493
  };
1494
+ this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
1305
1495
  if (this.next) {
1306
1496
  // now run any other transactions
1307
- options.timestamp = this.timestamp;
1497
+ options.timestamp = txnResolution.txnTime;
1308
1498
  // as above: the next store must not inherit this store's explicit native transaction
1309
1499
  let nextResolution;
1310
1500
  try {
@@ -1323,7 +1513,7 @@ export class DatabaseTransaction implements Transaction {
1323
1513
  // scope resumable after a partially failed mid-scope commit.
1324
1514
  this.completeMidScopeCommit(options);
1325
1515
  return {
1326
- txnTime: this.timestamp,
1516
+ txnTime: txnResolution.txnTime,
1327
1517
  next: nextResolution,
1328
1518
  };
1329
1519
  },
@@ -1342,6 +1532,7 @@ export class DatabaseTransaction implements Transaction {
1342
1532
  return txnResolution;
1343
1533
  },
1344
1534
  (error) => {
1535
+ this.setCommitPhase(false);
1345
1536
  this.abort();
1346
1537
  throw error;
1347
1538
  }
@@ -1390,6 +1581,7 @@ export class DatabaseTransaction implements Transaction {
1390
1581
  // Defensively release any native handle whose reference bookkeeping was already consumed.
1391
1582
  if (this.transaction) this.releaseReadTxn();
1392
1583
  this.open = TRANSACTION_STATE.CLOSED;
1584
+ this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
1393
1585
  this.drainCompletions();
1394
1586
  try {
1395
1587
  for (const write of this.writes) {
@@ -1399,6 +1591,7 @@ export class DatabaseTransaction implements Transaction {
1399
1591
  } finally {
1400
1592
  this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
1401
1593
  this.clearWrites();
1594
+ this.releaseRecordLocks();
1402
1595
  // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
1403
1596
  // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
1404
1597
  // starting a fresh one) so the rest of the logical operation fails atomically via the
@@ -1417,6 +1610,62 @@ export class DatabaseTransaction implements Transaction {
1417
1610
  }
1418
1611
  }
1419
1612
  }
1613
+ /** How long this logical commit may keep retrying: the thread-wide queue limit, or its own larger explicit budget. */
1614
+ private commitConflictBudget(): number {
1615
+ return Math.max(MAX_OUTSTANDING_TXN_DURATION, (this.root ?? this).timeoutBudget || 0);
1616
+ }
1617
+
1618
+ /**
1619
+ * How long this logical commit has been retrying once it is past its budget, else 0. rocksdb-js
1620
+ * returns control from a parked commit every `ROCKSDB_JS_PARK_TIMEOUT_MS` even when the intent
1621
+ * holder never releases, so without this bound a request-path commit retries the attempt cap out
1622
+ * — minutes past the queue limit an operator configured (issue #2450).
1623
+ *
1624
+ * Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
1625
+ * there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
1626
+ */
1627
+ private elapsedPastCommitBudget(): number {
1628
+ if (this.sourceApply) return 0;
1629
+ const startedAt = (this.root ?? this).commitStartedAt;
1630
+ if (startedAt == null) return 0;
1631
+ const elapsed = performance.now() - startedAt;
1632
+ return elapsed > this.commitConflictBudget() ? elapsed : 0;
1633
+ }
1634
+
1635
+ /**
1636
+ * Abandon a logical commit that stayed in write-intent conflict past its budget. Cleanup is
1637
+ * retry exhaustion's, so no link leaks a native handle or read snapshot; the error is distinct
1638
+ * because the condition is distinct — every attempt reported transient contention, so a later
1639
+ * request can succeed once the holder releases, which the generic exhaustion 500 does not say.
1640
+ *
1641
+ * Only a chain root may report `retryable`: a link commits solely from its predecessor's success
1642
+ * handler, so anywhere else in the chain an earlier store has already landed durable audit
1643
+ * entries and hooks that a replayed request would run twice. A head whose scope already rotated
1644
+ * through a mid-scope commit is in the same position.
1645
+ */
1646
+ private abandonCommitAfterDeadline(headTransaction: RocksTransaction, elapsedMs: number): never {
1647
+ const elapsed = Math.round(elapsedMs);
1648
+ const budget = this.commitConflictBudget();
1649
+ const retryable = !this.root && !this.snapshotFree;
1650
+ if (allowStuckCommitLog('abandon', performance.now())) {
1651
+ harperLogger.error(
1652
+ `Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
1653
+ `(exceeds the ${budget}ms limit) across ${this.retries} retries, ` +
1654
+ describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
1655
+ `.` +
1656
+ describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransaction?.id) +
1657
+ ` Another transaction holds a conflicting write intent and has not completed; the request is ` +
1658
+ `failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
1659
+ `rather than waiting further.`
1660
+ );
1661
+ }
1662
+ this.abortChainAfterRetries(headTransaction);
1663
+ throw new TransactionCommitConflictTimeoutError(
1664
+ `Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${this.retries} retries`,
1665
+ retryable
1666
+ );
1667
+ }
1668
+
1420
1669
  /**
1421
1670
  * Give up on a chain of linked transactions after exhausting conflict retries: poison every link
1422
1671
  * first, then abort each link's native transaction and release its DatabaseTransaction-level
@@ -1506,11 +1755,7 @@ export class DatabaseTransaction implements Transaction {
1506
1755
  // throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
1507
1756
  // staged.
1508
1757
  this.detachOwnedTransaction();
1509
- try {
1510
- transaction?.abort();
1511
- } catch (abortError) {
1512
- harperLogger.debug?.('aborting a transaction whose synchronous commit failed', abortError);
1513
- }
1758
+ abortNativeTransaction(transaction, 'aborting a transaction whose synchronous commit failed');
1514
1759
  try {
1515
1760
  this.abort();
1516
1761
  } catch (abortError) {
@@ -1537,6 +1782,20 @@ export interface Transaction {
1537
1782
  abort?(): any;
1538
1783
  }
1539
1784
 
1785
+ export function shouldSpareCommitPhase(
1786
+ txn: DatabaseTransaction,
1787
+ checkedCommitPhaseChains: Set<DatabaseTransaction>
1788
+ ): boolean {
1789
+ if (!txn.committing) return false;
1790
+ if (txn.sourceApply || txn.isReplay) return true;
1791
+ const commitChainHead = txn.commitChainHead ?? txn;
1792
+ if (!checkedCommitPhaseChains.has(commitChainHead)) {
1793
+ checkedCommitPhaseChains.add(commitChainHead);
1794
+ commitChainHead.commitPhaseTicks++;
1795
+ }
1796
+ return commitChainHead.commitPhaseTicks <= COMMIT_PHASE_GRACE;
1797
+ }
1798
+
1540
1799
  export class ImmediateTransaction extends DatabaseTransaction {
1541
1800
  isCommitting = false;
1542
1801
  saveCommits = true;
@@ -1545,16 +1804,47 @@ export class ImmediateTransaction extends DatabaseTransaction {
1545
1804
  this.db = db;
1546
1805
  }
1547
1806
  save(...args: any[]): any {
1548
- const transaction = args[0];
1807
+ const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
1549
1808
  if (this.isCommitting) {
1550
1809
  // if we are in the commit, do the save and force a reload so we get a read within the transaction
1551
- super.save(transaction, null as any, true);
1810
+ super.save(operation, null as any, true);
1552
1811
  } else {
1553
1812
  this.isCommitting = true;
1554
- return when(this.commit(), () => {
1813
+ // A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
1814
+ // otherwise leave isCommitting latched at true, causing every subsequent save() in this
1815
+ // context to take the fire-and-forget if-branch and silently drop writes.
1816
+ let commitResult: any;
1817
+ try {
1818
+ commitResult = this.commit();
1819
+ } catch (err) {
1555
1820
  this.isCommitting = false;
1556
- });
1821
+ throw err;
1822
+ }
1823
+ return when(
1824
+ commitResult,
1825
+ () => {
1826
+ this.isCommitting = false;
1827
+ },
1828
+ (err: any) => {
1829
+ // Async rejection (e.g. a rejected rocksdb commit promise) must also clear the
1830
+ // latch; without this, every subsequent save() silently fire-and-forgets.
1831
+ this.isCommitting = false;
1832
+ throw err;
1833
+ }
1834
+ );
1835
+ }
1836
+ }
1837
+
1838
+ // Without an explicit transaction() a live lock is released only by unlock() or its lease, never
1839
+ // by the per-write commit. Expired handles are pruned so a reused context does not retain every key.
1840
+ releaseRecordLocks(): void {
1841
+ const recordLocks = this.recordLocks;
1842
+ if (!recordLocks) return;
1843
+ for (const [store, storeMap] of recordLocks) {
1844
+ for (const [keyId, handle] of storeMap) if (handle.released) storeMap.delete(keyId);
1845
+ if (storeMap.size === 0) recordLocks.delete(store);
1557
1846
  }
1847
+ if (recordLocks.size === 0) this.recordLocks = undefined;
1558
1848
  }
1559
1849
 
1560
1850
  declare _timestamp: number;
@@ -1635,6 +1925,7 @@ export function isJoinableScope(transaction: DatabaseTransaction | null | undefi
1635
1925
  }
1636
1926
 
1637
1927
  let timer;
1928
+ let monitorTick = 0;
1638
1929
 
1639
1930
  /**
1640
1931
  * True when a link other than `txn` in the same multi-store chain was written recently enough to
@@ -1654,20 +1945,96 @@ function chainStillActive(txn: DatabaseTransaction): boolean {
1654
1945
  return false;
1655
1946
  }
1656
1947
 
1948
+ /**
1949
+ * Name a chain link still holding its native handle past the reporting threshold, and why the
1950
+ * branches below are not reaping it (harper#2471). Attribution only: it reads the recency clocks
1951
+ * those branches maintain rather than calling `chainStillActive`, which would decay `writeTimeout`
1952
+ * as a side effect, and it runs before them so it cannot reorder them.
1953
+ */
1954
+ function reportLongLivedLink(
1955
+ link: DatabaseTransaction,
1956
+ thresholdMs: number,
1957
+ now: number,
1958
+ reportBudget: LongLivedHolderReportBudget
1959
+ ): void {
1960
+ if (!link.transaction || link.handleOpenedAt === 0) return;
1961
+ const ageMs = now - link.handleOpenedAt;
1962
+ if (ageMs < thresholdMs) return;
1963
+ const states: string[] = [];
1964
+ if (link.sourceApply) states.push('source-apply');
1965
+ if (link.isReplay) states.push('replay');
1966
+ if (link.committing) states.push('commit-phase');
1967
+ if (link.open === TRANSACTION_STATE.CLOSED) states.push('closed-with-open-iterators');
1968
+ if (monitorTick - link.writeTick <= 1) states.push('active');
1969
+ if (states.length === 0) states.push('over-limit');
1970
+ reportLongLivedHolder(
1971
+ {
1972
+ databasePath: (link.db as any)?.rootStore?.path,
1973
+ nativeId: link.transaction.id,
1974
+ openedAt: link.handleOpenedAt,
1975
+ ageMs,
1976
+ databaseName: (link.db as any)?.rootStore?.databaseName,
1977
+ tableName: (link.db as any)?.name,
1978
+ countPendingWrites: () => {
1979
+ let pendingWrites = 0;
1980
+ for (const write of link.writes) if (write) pendingWrites++;
1981
+ return pendingWrites;
1982
+ },
1983
+ states,
1984
+ timeoutBudget: link.timeoutBudget,
1985
+ startedFrom: link.startedFrom,
1986
+ },
1987
+ reportBudget
1988
+ );
1989
+ }
1990
+
1991
+ /**
1992
+ * Report every link of this logical transaction that holds its own native handle. A blind write to a
1993
+ * second database attaches a handle to a `.next` link while only the root is supervised, so reporting
1994
+ * the entry alone names an id the sweep's line can never be joined to when the child is the holder.
1995
+ * Links that read are in `trackedTxns` and get their own visit from the tick.
1996
+ */
1997
+ function reportIfLongLived(
1998
+ txn: DatabaseTransaction,
1999
+ thresholdMs: number,
2000
+ now: number,
2001
+ reportBudget: LongLivedHolderReportBudget
2002
+ ): void {
2003
+ if (thresholdMs === 0) return;
2004
+ for (let link: DatabaseTransaction = txn; link; link = link.next) {
2005
+ if (link !== txn && trackedTxns.has(link)) break;
2006
+ reportLongLivedLink(link, thresholdMs, now, reportBudget);
2007
+ }
2008
+ }
2009
+
1657
2010
  function startMonitoringTxns() {
1658
2011
  timer = setInterval(function () {
2012
+ monitorTick++;
2013
+ const checkedCommitPhaseChains = new Set<DatabaseTransaction>();
2014
+ const reportThresholdMs = getReportThresholdMs();
2015
+ rebaseLongLivedTransactionReports(reportThresholdMs);
2016
+ const reportBudget = createLongLivedHolderReportBudget(reportThresholdMs);
2017
+ const reportNow = performance.now();
1659
2018
  // Both registries, in sequence rather than as a union: a root can be in each (it read, and a
1660
2019
  // later link blind-wrote), and the membership check is cheaper than allocating per tick.
1661
- for (const txn of trackedTxns) monitorTransaction(txn);
1662
- for (const txn of supervisedWriteRoots) if (!trackedTxns.has(txn)) monitorTransaction(txn);
2020
+ for (const txn of trackedTxns)
2021
+ monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
2022
+ for (const txn of supervisedWriteRoots)
2023
+ if (!trackedTxns.has(txn))
2024
+ monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
2025
+ finishLongLivedHolderReports(reportBudget);
1663
2026
  }, txnExpiration).unref();
1664
2027
 
1665
- function monitorTransaction(txn: DatabaseTransaction) {
1666
- if (txn.rangeReadActive) {
1667
- txn.rangeReadActive = false;
1668
- txn.renewReadTimeout();
1669
- }
2028
+ function monitorTransaction(
2029
+ txn: DatabaseTransaction,
2030
+ checkedCommitPhaseChains: Set<DatabaseTransaction>,
2031
+ reportThresholdMs: number,
2032
+ reportNow: number,
2033
+ reportBudget: LongLivedHolderReportBudget
2034
+ ) {
2035
+ reportIfLongLived(txn, reportThresholdMs, reportNow, reportBudget);
1670
2036
  {
2037
+ const commitChainHead = txn.commitChainHead ?? txn;
1671
2038
  // Decay write recency once per tick for every tracked link, independent of the `timeout`
1672
2039
  // branches below — a tracked link that keeps its own idle limit alive by reading must not
1673
2040
  // thereby keep chainStillActive believing it was written recently too.
@@ -1695,6 +2062,25 @@ function startMonitoringTxns() {
1695
2062
  }`
1696
2063
  );
1697
2064
  txn.releaseReadTxn();
2065
+ } else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
2066
+ // Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a
2067
+ // blob's durable file write, which for a multi-tens-of-MB payload legitimately outruns the
2068
+ // limit. The write set is sealed and the caller is awaiting this commit, so the limit's
2069
+ // premise (the application is holding a transaction open) does not hold, and poisoning here
2070
+ // would unlink the blobs the write still references (issue #2062). The grace is bounded
2071
+ // because the transaction still pins a read snapshot: a source that stalls rather than
2072
+ // finishing falls through to the abort below once it runs out. A canonical-source apply or
2073
+ // replay is spared for as long as it takes: neither aborting it (harper-pro#348) nor the
2074
+ // force-commit below — which would durably commit a record whose blob file is still being
2075
+ // written — is acceptable, and their blob sources are bounded by the receive-side idle
2076
+ // watchdog instead.
2077
+ harperLogger.warn?.(
2078
+ `Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work (e.g. a large blob write); letting it complete, from table: ${
2079
+ (txn.db as any)?.name + (url ? ' path: ' + url : '')
2080
+ }`,
2081
+ ...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : [])
2082
+ );
2083
+ txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
1698
2084
  } else if (txn.hasPendingWrites() && chainStillActive(txn)) {
1699
2085
  // A later link in the chain was written recently (writes re-arm only the link that
1700
2086
  // receives them, and a multi-store transaction can be writing database B while this
@@ -1718,7 +2104,7 @@ function startMonitoringTxns() {
1718
2104
  ...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : [])
1719
2105
  );
1720
2106
  try {
1721
- txn.abortDueToTimeout();
2107
+ commitChainHead.abortDueToTimeout();
1722
2108
  } catch (error) {
1723
2109
  harperLogger.debug?.(`Error aborting timed out transaction: ${error.message}`);
1724
2110
  }
@@ -1756,6 +2142,12 @@ export function resetReplayedWritesWarning() {
1756
2142
  replayedWritesWarned = false;
1757
2143
  }
1758
2144
 
2145
+ export function setMaxOutstandingTxnDuration(ms: number): number {
2146
+ const previous = MAX_OUTSTANDING_TXN_DURATION;
2147
+ MAX_OUTSTANDING_TXN_DURATION = ms;
2148
+ return previous;
2149
+ }
2150
+
1759
2151
  /** Test seam: whether the monitor supervises this logical transaction for its writes. */
1760
2152
  export function isWriteSupervised(txn: DatabaseTransaction): boolean {
1761
2153
  return supervisedWriteRoots.has(txn);