@harperfast/harper 5.2.10 → 5.3.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (554) hide show
  1. package/agent/mcpTools.ts +1 -1
  2. package/agent/session.ts +25 -14
  3. package/bin/cliOperations.ts +46 -9
  4. package/bin/copyDb.ts +282 -59
  5. package/bin/deploySetup.ts +16 -5
  6. package/bin/harper.ts +1 -1
  7. package/bin/help.ts +4 -1
  8. package/bin/lite.ts +4 -1
  9. package/bin/restart.ts +59 -3
  10. package/bin/run.ts +6 -11
  11. package/bin/upgrade.js +7 -3
  12. package/bin/workloadIdentity.ts +119 -0
  13. package/components/Application.ts +2414 -242
  14. package/components/ApplicationScope.ts +8 -0
  15. package/components/EntryHandler.ts +59 -39
  16. package/components/OptionsWatcher.ts +150 -74
  17. package/components/RuntimeModuleTracker.ts +38 -7
  18. package/components/Scope.ts +37 -15
  19. package/components/awaitRestart.ts +84 -0
  20. package/components/componentLoader.ts +340 -30
  21. package/components/componentPreparationLock.ts +16 -5
  22. package/components/mcp/adapters/harperHttp.ts +4 -0
  23. package/components/mcp/listChanged.ts +4 -0
  24. package/components/mcp/toolRegistry.ts +2 -0
  25. package/components/mcp/tools/operations.ts +9 -0
  26. package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
  27. package/components/operations.js +270 -78
  28. package/components/operationsValidation.js +49 -1
  29. package/components/status/ComponentStatusRegistry.ts +59 -0
  30. package/config/RootConfigWatcher.ts +80 -34
  31. package/config/configUtils.ts +291 -34
  32. package/config/harperConfigEnvVars.ts +170 -27
  33. package/config-root.schema.json +29 -0
  34. package/dataLayer/blobBackup.ts +160 -50
  35. package/dataLayer/delete.ts +6 -1
  36. package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
  37. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  38. package/dataLayer/hdbInfoController.ts +34 -1
  39. package/dataLayer/insert.ts +44 -1
  40. package/dataLayer/rocksdbBackup.ts +53 -10
  41. package/dataLayer/schema.ts +11 -1
  42. package/dataLayer/schemaDescribe.ts +8 -1
  43. package/dist/agent/mcpTools.js +1 -1
  44. package/dist/agent/mcpTools.js.map +1 -1
  45. package/dist/agent/session.d.ts +22 -0
  46. package/dist/agent/session.js +26 -15
  47. package/dist/agent/session.js.map +1 -1
  48. package/dist/bin/cliOperations.js +48 -9
  49. package/dist/bin/cliOperations.js.map +1 -1
  50. package/dist/bin/copyDb.d.ts +12 -1
  51. package/dist/bin/copyDb.js +248 -60
  52. package/dist/bin/copyDb.js.map +1 -1
  53. package/dist/bin/deploySetup.d.ts +2 -0
  54. package/dist/bin/deploySetup.js +11 -3
  55. package/dist/bin/deploySetup.js.map +1 -1
  56. package/dist/bin/harper.js +1 -1
  57. package/dist/bin/harper.js.map +1 -1
  58. package/dist/bin/help.js +4 -1
  59. package/dist/bin/help.js.map +1 -1
  60. package/dist/bin/lite.js +4 -1
  61. package/dist/bin/lite.js.map +1 -1
  62. package/dist/bin/restart.js +54 -5
  63. package/dist/bin/restart.js.map +1 -1
  64. package/dist/bin/run.js +4 -10
  65. package/dist/bin/run.js.map +1 -1
  66. package/dist/bin/upgrade.js +4 -3
  67. package/dist/bin/upgrade.js.map +1 -1
  68. package/dist/bin/workloadIdentity.d.ts +18 -0
  69. package/dist/bin/workloadIdentity.js +100 -0
  70. package/dist/bin/workloadIdentity.js.map +1 -0
  71. package/dist/components/Application.d.ts +139 -16
  72. package/dist/components/Application.js +2215 -267
  73. package/dist/components/Application.js.map +1 -1
  74. package/dist/components/ApplicationScope.d.ts +8 -0
  75. package/dist/components/ApplicationScope.js +7 -0
  76. package/dist/components/ApplicationScope.js.map +1 -1
  77. package/dist/components/EntryHandler.js +26 -10
  78. package/dist/components/EntryHandler.js.map +1 -1
  79. package/dist/components/OptionsWatcher.d.ts +1 -0
  80. package/dist/components/OptionsWatcher.js +141 -74
  81. package/dist/components/OptionsWatcher.js.map +1 -1
  82. package/dist/components/RuntimeModuleTracker.js +40 -6
  83. package/dist/components/RuntimeModuleTracker.js.map +1 -1
  84. package/dist/components/Scope.js +38 -13
  85. package/dist/components/Scope.js.map +1 -1
  86. package/dist/components/awaitRestart.d.ts +33 -0
  87. package/dist/components/awaitRestart.js +61 -0
  88. package/dist/components/awaitRestart.js.map +1 -0
  89. package/dist/components/componentLoader.d.ts +38 -1
  90. package/dist/components/componentLoader.js +279 -22
  91. package/dist/components/componentLoader.js.map +1 -1
  92. package/dist/components/componentPreparationLock.d.ts +5 -0
  93. package/dist/components/componentPreparationLock.js +14 -6
  94. package/dist/components/componentPreparationLock.js.map +1 -1
  95. package/dist/components/mcp/adapters/harperHttp.js +4 -0
  96. package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
  97. package/dist/components/mcp/listChanged.js +5 -0
  98. package/dist/components/mcp/listChanged.js.map +1 -1
  99. package/dist/components/mcp/toolRegistry.d.ts +1 -0
  100. package/dist/components/mcp/toolRegistry.js.map +1 -1
  101. package/dist/components/mcp/tools/operations.d.ts +5 -0
  102. package/dist/components/mcp/tools/operations.js +9 -0
  103. package/dist/components/mcp/tools/operations.js.map +1 -1
  104. package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
  105. package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
  106. package/dist/components/operations.js +231 -77
  107. package/dist/components/operations.js.map +1 -1
  108. package/dist/components/operationsValidation.js +49 -1
  109. package/dist/components/operationsValidation.js.map +1 -1
  110. package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
  111. package/dist/components/status/ComponentStatusRegistry.js +63 -0
  112. package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
  113. package/dist/config/RootConfigWatcher.d.ts +7 -1
  114. package/dist/config/RootConfigWatcher.js +64 -27
  115. package/dist/config/RootConfigWatcher.js.map +1 -1
  116. package/dist/config/configUtils.d.ts +9 -1
  117. package/dist/config/configUtils.js +254 -33
  118. package/dist/config/configUtils.js.map +1 -1
  119. package/dist/config/harperConfigEnvVars.d.ts +16 -0
  120. package/dist/config/harperConfigEnvVars.js +162 -25
  121. package/dist/config/harperConfigEnvVars.js.map +1 -1
  122. package/dist/dataLayer/blobBackup.d.ts +49 -20
  123. package/dist/dataLayer/blobBackup.js +139 -50
  124. package/dist/dataLayer/blobBackup.js.map +1 -1
  125. package/dist/dataLayer/delete.js +1 -1
  126. package/dist/dataLayer/delete.js.map +1 -1
  127. package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
  128. package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
  129. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  130. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
  131. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  132. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
  133. package/dist/dataLayer/hdbInfoController.d.ts +10 -0
  134. package/dist/dataLayer/hdbInfoController.js +30 -1
  135. package/dist/dataLayer/hdbInfoController.js.map +1 -1
  136. package/dist/dataLayer/insert.d.ts +9 -1
  137. package/dist/dataLayer/insert.js +30 -0
  138. package/dist/dataLayer/insert.js.map +1 -1
  139. package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
  140. package/dist/dataLayer/rocksdbBackup.js +45 -8
  141. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  142. package/dist/dataLayer/schema.js +8 -0
  143. package/dist/dataLayer/schema.js.map +1 -1
  144. package/dist/dataLayer/schemaDescribe.js +8 -1
  145. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  146. package/dist/index.d.ts +1 -1
  147. package/dist/index.js.map +1 -1
  148. package/dist/json/systemSchema.json +52 -0
  149. package/dist/resources/DatabaseTransaction.d.ts +55 -3
  150. package/dist/resources/DatabaseTransaction.js +460 -134
  151. package/dist/resources/DatabaseTransaction.js.map +1 -1
  152. package/dist/resources/LMDBTransaction.js +22 -4
  153. package/dist/resources/LMDBTransaction.js.map +1 -1
  154. package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
  155. package/dist/resources/PrimaryRocksDatabase.js +30 -2
  156. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  157. package/dist/resources/RecordEncoder.d.ts +20 -0
  158. package/dist/resources/RecordEncoder.js +94 -9
  159. package/dist/resources/RecordEncoder.js.map +1 -1
  160. package/dist/resources/RequestTarget.d.ts +2 -0
  161. package/dist/resources/RequestTarget.js.map +1 -1
  162. package/dist/resources/Resource.js +20 -11
  163. package/dist/resources/Resource.js.map +1 -1
  164. package/dist/resources/ResourceInterface.d.ts +20 -1
  165. package/dist/resources/ResourceInterface.js.map +1 -1
  166. package/dist/resources/RocksIndexStore.d.ts +6 -1
  167. package/dist/resources/RocksIndexStore.js +24 -10
  168. package/dist/resources/RocksIndexStore.js.map +1 -1
  169. package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
  170. package/dist/resources/RocksTransactionLogStore.js +57 -17
  171. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  172. package/dist/resources/Table.d.ts +112 -8
  173. package/dist/resources/Table.js +1071 -197
  174. package/dist/resources/Table.js.map +1 -1
  175. package/dist/resources/auditStore.d.ts +11 -2
  176. package/dist/resources/auditStore.js +183 -18
  177. package/dist/resources/auditStore.js.map +1 -1
  178. package/dist/resources/blob.d.ts +129 -9
  179. package/dist/resources/blob.js +992 -114
  180. package/dist/resources/blob.js.map +1 -1
  181. package/dist/resources/branchDatabase.d.ts +48 -0
  182. package/dist/resources/branchDatabase.js +892 -0
  183. package/dist/resources/branchDatabase.js.map +1 -0
  184. package/dist/resources/crdt.js +50 -12
  185. package/dist/resources/crdt.js.map +1 -1
  186. package/dist/resources/databases.d.ts +150 -9
  187. package/dist/resources/databases.js +1247 -525
  188. package/dist/resources/databases.js.map +1 -1
  189. package/dist/resources/defineTable.d.ts +10 -2
  190. package/dist/resources/defineTable.js +9 -1
  191. package/dist/resources/defineTable.js.map +1 -1
  192. package/dist/resources/graphql.d.ts +1 -1
  193. package/dist/resources/graphql.js +50 -15
  194. package/dist/resources/graphql.js.map +1 -1
  195. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
  196. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
  197. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  198. package/dist/resources/longLivedTransactions.d.ts +71 -0
  199. package/dist/resources/longLivedTransactions.js +358 -0
  200. package/dist/resources/longLivedTransactions.js.map +1 -0
  201. package/dist/resources/models/backendRegistry.d.ts +26 -0
  202. package/dist/resources/models/backendRegistry.js +60 -2
  203. package/dist/resources/models/backendRegistry.js.map +1 -1
  204. package/dist/resources/models/bootstrap.d.ts +33 -1
  205. package/dist/resources/models/bootstrap.js +416 -31
  206. package/dist/resources/models/bootstrap.js.map +1 -1
  207. package/dist/resources/recordLock.d.ts +80 -0
  208. package/dist/resources/recordLock.js +184 -0
  209. package/dist/resources/recordLock.js.map +1 -0
  210. package/dist/resources/replayLogs.d.ts +14 -1
  211. package/dist/resources/replayLogs.js +152 -24
  212. package/dist/resources/replayLogs.js.map +1 -1
  213. package/dist/resources/replayLogsGuards.d.ts +94 -7
  214. package/dist/resources/replayLogsGuards.js +111 -7
  215. package/dist/resources/replayLogsGuards.js.map +1 -1
  216. package/dist/resources/search.js +158 -32
  217. package/dist/resources/search.js.map +1 -1
  218. package/dist/resources/transactionBroadcast.d.ts +1 -1
  219. package/dist/resources/transactionBroadcast.js +2 -2
  220. package/dist/resources/transactionBroadcast.js.map +1 -1
  221. package/dist/security/auth.js +61 -30
  222. package/dist/security/auth.js.map +1 -1
  223. package/dist/security/authn/oidc/claims.d.ts +22 -0
  224. package/dist/security/authn/oidc/claims.js +71 -0
  225. package/dist/security/authn/oidc/claims.js.map +1 -0
  226. package/dist/security/authn/oidc/identityToken.d.ts +27 -0
  227. package/dist/security/authn/oidc/identityToken.js +111 -0
  228. package/dist/security/authn/oidc/identityToken.js.map +1 -0
  229. package/dist/security/authn/oidc/jwks.d.ts +25 -0
  230. package/dist/security/authn/oidc/jwks.js +261 -0
  231. package/dist/security/authn/oidc/jwks.js.map +1 -0
  232. package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
  233. package/dist/security/authn/oidc/providers/generic.js +34 -0
  234. package/dist/security/authn/oidc/providers/generic.js.map +1 -0
  235. package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
  236. package/dist/security/authn/oidc/providers/githubActions.js +129 -0
  237. package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
  238. package/dist/security/authn/oidc/providers/index.d.ts +37 -0
  239. package/dist/security/authn/oidc/providers/index.js +24 -0
  240. package/dist/security/authn/oidc/providers/index.js.map +1 -0
  241. package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
  242. package/dist/security/authn/oidc/tokenExchange.js +306 -0
  243. package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
  244. package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
  245. package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
  246. package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
  247. package/dist/security/authn/oidc/types.d.ts +38 -0
  248. package/dist/security/authn/oidc/types.js +6 -0
  249. package/dist/security/authn/oidc/types.js.map +1 -0
  250. package/dist/security/certificateVerification/index.js +40 -11
  251. package/dist/security/certificateVerification/index.js.map +1 -1
  252. package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
  253. package/dist/security/certificateVerification/trustedIssuers.js +79 -0
  254. package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
  255. package/dist/security/certificateVerification/types.d.ts +1 -0
  256. package/dist/security/credentialProvenance.d.ts +35 -0
  257. package/dist/security/credentialProvenance.js +51 -0
  258. package/dist/security/credentialProvenance.js.map +1 -0
  259. package/dist/security/credentialRejection.d.ts +4 -0
  260. package/dist/security/credentialRejection.js +24 -0
  261. package/dist/security/credentialRejection.js.map +1 -0
  262. package/dist/security/deferredAuthentication.d.ts +36 -0
  263. package/dist/security/deferredAuthentication.js +70 -0
  264. package/dist/security/deferredAuthentication.js.map +1 -0
  265. package/dist/security/impersonation.d.ts +21 -0
  266. package/dist/security/impersonation.js +108 -9
  267. package/dist/security/impersonation.js.map +1 -1
  268. package/dist/security/jsLoader.d.ts +6 -0
  269. package/dist/security/jsLoader.js +75 -15
  270. package/dist/security/jsLoader.js.map +1 -1
  271. package/dist/security/keys.js +301 -71
  272. package/dist/security/keys.js.map +1 -1
  273. package/dist/security/operationScope.d.ts +21 -0
  274. package/dist/security/operationScope.js +36 -0
  275. package/dist/security/operationScope.js.map +1 -0
  276. package/dist/security/permissionsTranslator.js +21 -0
  277. package/dist/security/permissionsTranslator.js.map +1 -1
  278. package/dist/security/tokenAuthentication.d.ts +19 -1
  279. package/dist/security/tokenAuthentication.js +191 -10
  280. package/dist/security/tokenAuthentication.js.map +1 -1
  281. package/dist/security/user.js +4 -3
  282. package/dist/security/user.js.map +1 -1
  283. package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
  284. package/dist/server/DurableSubscriptionsSession.js +67 -10
  285. package/dist/server/DurableSubscriptionsSession.js.map +1 -1
  286. package/dist/server/REST.js +73 -0
  287. package/dist/server/REST.js.map +1 -1
  288. package/dist/server/graphqlQuerying.js +4 -0
  289. package/dist/server/graphqlQuerying.js.map +1 -1
  290. package/dist/server/http.d.ts +11 -0
  291. package/dist/server/http.js +88 -15
  292. package/dist/server/http.js.map +1 -1
  293. package/dist/server/itc/serverHandlers.js +8 -1
  294. package/dist/server/itc/serverHandlers.js.map +1 -1
  295. package/dist/server/jobs/jobProcess.js +6 -2
  296. package/dist/server/jobs/jobProcess.js.map +1 -1
  297. package/dist/server/jobs/jobs.js +4 -1
  298. package/dist/server/jobs/jobs.js.map +1 -1
  299. package/dist/server/liveSubscriptionAuth.d.ts +26 -4
  300. package/dist/server/liveSubscriptionAuth.js +105 -39
  301. package/dist/server/liveSubscriptionAuth.js.map +1 -1
  302. package/dist/server/loadRootComponents.js +49 -10
  303. package/dist/server/loadRootComponents.js.map +1 -1
  304. package/dist/server/mqtt.d.ts +2 -0
  305. package/dist/server/mqtt.js +165 -30
  306. package/dist/server/mqtt.js.map +1 -1
  307. package/dist/server/nodeName.d.ts +2 -0
  308. package/dist/server/nodeName.js +107 -23
  309. package/dist/server/nodeName.js.map +1 -1
  310. package/dist/server/serverHelpers/Headers.d.ts +25 -0
  311. package/dist/server/serverHelpers/Headers.js +139 -1
  312. package/dist/server/serverHelpers/Headers.js.map +1 -1
  313. package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
  314. package/dist/server/serverHelpers/contentTypes.js +32 -25
  315. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  316. package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
  317. package/dist/server/serverHelpers/deployValidationState.js +9 -19
  318. package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
  319. package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
  320. package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
  321. package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
  322. package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
  323. package/dist/server/serverHelpers/registeredOperations.js +74 -21
  324. package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
  325. package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
  326. package/dist/server/serverHelpers/requestSanitization.js +20 -0
  327. package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
  328. package/dist/server/serverHelpers/serverHandlers.js +6 -3
  329. package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
  330. package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
  331. package/dist/server/serverHelpers/serverUtilities.js +90 -20
  332. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  333. package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
  334. package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
  335. package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
  336. package/dist/server/serverHelpers/uwsServer.js +19 -1
  337. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  338. package/dist/server/static.js +24 -28
  339. package/dist/server/static.js.map +1 -1
  340. package/dist/server/storageReclamation.d.ts +5 -0
  341. package/dist/server/storageReclamation.js +16 -0
  342. package/dist/server/storageReclamation.js.map +1 -1
  343. package/dist/server/threads/itc.d.ts +7 -2
  344. package/dist/server/threads/itc.js +5 -1
  345. package/dist/server/threads/itc.js.map +1 -1
  346. package/dist/server/threads/manageThreads.d.ts +26 -4
  347. package/dist/server/threads/manageThreads.js +517 -72
  348. package/dist/server/threads/manageThreads.js.map +1 -1
  349. package/dist/server/threads/socketRouter.js +89 -26
  350. package/dist/server/threads/socketRouter.js.map +1 -1
  351. package/dist/server/threads/threadHeapMemory.d.ts +2 -0
  352. package/dist/server/threads/threadHeapMemory.js +31 -0
  353. package/dist/server/threads/threadHeapMemory.js.map +1 -0
  354. package/dist/server/threads/threadServer.js +46 -15
  355. package/dist/server/threads/threadServer.js.map +1 -1
  356. package/dist/sqlEngine/config.d.ts +1 -3
  357. package/dist/sqlEngine/config.js +19 -16
  358. package/dist/sqlEngine/config.js.map +1 -1
  359. package/dist/sqlTranslator/index.d.ts +1 -1
  360. package/dist/sqlTranslator/index.js +30 -7
  361. package/dist/sqlTranslator/index.js.map +1 -1
  362. package/dist/upgrade/directives/5-3-0.d.ts +7 -0
  363. package/dist/upgrade/directives/5-3-0.js +148 -0
  364. package/dist/upgrade/directives/5-3-0.js.map +1 -0
  365. package/dist/upgrade/directives/directivesController.js +2 -1
  366. package/dist/upgrade/directives/directivesController.js.map +1 -1
  367. package/dist/utility/OperationFunctionCaller.js +2 -1
  368. package/dist/utility/OperationFunctionCaller.js.map +1 -1
  369. package/dist/utility/common_utils.d.ts +16 -0
  370. package/dist/utility/common_utils.js +32 -6
  371. package/dist/utility/common_utils.js.map +1 -1
  372. package/dist/utility/componentNames.d.ts +8 -0
  373. package/dist/utility/componentNames.js +12 -1
  374. package/dist/utility/componentNames.js.map +1 -1
  375. package/dist/utility/environment/environmentManager.js +3 -6
  376. package/dist/utility/environment/environmentManager.js.map +1 -1
  377. package/dist/utility/environment/systemInformation.d.ts +1 -0
  378. package/dist/utility/environment/systemInformation.js +1 -0
  379. package/dist/utility/environment/systemInformation.js.map +1 -1
  380. package/dist/utility/errors/commonErrors.d.ts +2 -0
  381. package/dist/utility/errors/commonErrors.js +2 -0
  382. package/dist/utility/errors/commonErrors.js.map +1 -1
  383. package/dist/utility/errors/hdbError.d.ts +17 -0
  384. package/dist/utility/errors/hdbError.js +30 -1
  385. package/dist/utility/errors/hdbError.js.map +1 -1
  386. package/dist/utility/globalSchema.d.ts +18 -0
  387. package/dist/utility/hdbTerms.d.ts +16 -0
  388. package/dist/utility/hdbTerms.js +18 -2
  389. package/dist/utility/hdbTerms.js.map +1 -1
  390. package/dist/utility/logging/harper_logger.d.ts +2 -0
  391. package/dist/utility/logging/harper_logger.js +91 -14
  392. package/dist/utility/logging/harper_logger.js.map +1 -1
  393. package/dist/utility/logging/logRotator.js +76 -49
  394. package/dist/utility/logging/logRotator.js.map +1 -1
  395. package/dist/utility/nodeIdentity.d.ts +9 -0
  396. package/dist/utility/nodeIdentity.js +58 -0
  397. package/dist/utility/nodeIdentity.js.map +1 -0
  398. package/dist/utility/npmUtilities.js +9 -7
  399. package/dist/utility/npmUtilities.js.map +1 -1
  400. package/dist/utility/operationPermissions.d.ts +3 -1
  401. package/dist/utility/operationPermissions.js +16 -1
  402. package/dist/utility/operationPermissions.js.map +1 -1
  403. package/dist/utility/operation_authorization.d.ts +10 -7
  404. package/dist/utility/operation_authorization.js +212 -41
  405. package/dist/utility/operation_authorization.js.map +1 -1
  406. package/dist/utility/watchPath.d.ts +29 -0
  407. package/dist/utility/watchPath.js +68 -0
  408. package/dist/utility/watchPath.js.map +1 -0
  409. package/dist/utility/watcherFallback.d.ts +86 -0
  410. package/dist/utility/watcherFallback.js +278 -1
  411. package/dist/utility/watcherFallback.js.map +1 -1
  412. package/dist/validation/configValidator.d.ts +12 -0
  413. package/dist/validation/configValidator.js +199 -72
  414. package/dist/validation/configValidator.js.map +1 -1
  415. package/dist/validation/installValidator.js +12 -0
  416. package/dist/validation/installValidator.js.map +1 -1
  417. package/dist/validation/validationWrapper.d.ts +11 -0
  418. package/dist/validation/validationWrapper.js +16 -3
  419. package/dist/validation/validationWrapper.js.map +1 -1
  420. package/index.ts +2 -0
  421. package/json/systemSchema.json +52 -0
  422. package/npm-shrinkwrap.json +235 -233
  423. package/package.json +8 -7
  424. package/resources/DESIGN.md +114 -52
  425. package/resources/DatabaseTransaction.ts +522 -130
  426. package/resources/LMDBTransaction.ts +26 -4
  427. package/resources/PrimaryRocksDatabase.ts +29 -3
  428. package/resources/RecordEncoder.ts +103 -8
  429. package/resources/RequestTarget.ts +2 -0
  430. package/resources/Resource.ts +17 -9
  431. package/resources/ResourceInterface.ts +23 -0
  432. package/resources/RocksIndexStore.ts +30 -10
  433. package/resources/RocksTransactionLogStore.ts +78 -21
  434. package/resources/Table.ts +1107 -164
  435. package/resources/auditStore.ts +187 -21
  436. package/resources/blob.ts +1029 -110
  437. package/resources/branchDatabase.ts +941 -0
  438. package/resources/crdt.ts +76 -12
  439. package/resources/databases.ts +1354 -522
  440. package/resources/defineTable.ts +18 -2
  441. package/resources/graphql.ts +70 -16
  442. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
  443. package/resources/longLivedTransactions.ts +360 -0
  444. package/resources/models/backendRegistry.ts +84 -2
  445. package/resources/models/bootstrap.ts +473 -28
  446. package/resources/recordLock.ts +253 -0
  447. package/resources/replayLogs.ts +147 -26
  448. package/resources/replayLogsGuards.ts +171 -8
  449. package/resources/search.ts +154 -32
  450. package/resources/transactionBroadcast.ts +3 -3
  451. package/security/auth.ts +68 -29
  452. package/security/authn/oidc/claims.ts +72 -0
  453. package/security/authn/oidc/identityToken.ts +129 -0
  454. package/security/authn/oidc/jwks.ts +260 -0
  455. package/security/authn/oidc/providers/generic.ts +40 -0
  456. package/security/authn/oidc/providers/githubActions.ts +137 -0
  457. package/security/authn/oidc/providers/index.ts +52 -0
  458. package/security/authn/oidc/tokenExchange.ts +300 -0
  459. package/security/authn/oidc/trustPolicyOperations.ts +343 -0
  460. package/security/authn/oidc/types.ts +41 -0
  461. package/security/certificateVerification/index.ts +54 -13
  462. package/security/certificateVerification/trustedIssuers.ts +76 -0
  463. package/security/certificateVerification/types.ts +1 -0
  464. package/security/credentialProvenance.ts +47 -0
  465. package/security/credentialRejection.ts +22 -0
  466. package/security/deferredAuthentication.ts +71 -0
  467. package/security/impersonation.ts +117 -12
  468. package/security/jsLoader.ts +81 -18
  469. package/security/keys.ts +298 -72
  470. package/security/operationScope.ts +33 -0
  471. package/security/permissionsTranslator.js +23 -0
  472. package/security/tokenAuthentication.ts +233 -12
  473. package/security/user.ts +4 -3
  474. package/server/DESIGN.md +183 -16
  475. package/server/DurableSubscriptionsSession.ts +71 -11
  476. package/server/REST.ts +79 -1
  477. package/server/graphqlQuerying.ts +4 -0
  478. package/server/http.ts +99 -18
  479. package/server/itc/serverHandlers.js +8 -1
  480. package/server/jobs/jobProcess.ts +8 -2
  481. package/server/jobs/jobs.ts +4 -1
  482. package/server/liveSubscriptionAuth.ts +129 -46
  483. package/server/loadRootComponents.js +50 -8
  484. package/server/mqtt.ts +179 -38
  485. package/server/nodeName.ts +103 -21
  486. package/server/serverHelpers/Headers.ts +131 -0
  487. package/server/serverHelpers/contentTypes.ts +29 -21
  488. package/server/serverHelpers/deployValidationState.ts +24 -13
  489. package/server/serverHelpers/operationAuthorizationState.ts +34 -3
  490. package/server/serverHelpers/registeredOperations.ts +79 -22
  491. package/server/serverHelpers/requestSanitization.ts +15 -0
  492. package/server/serverHelpers/serverHandlers.js +6 -3
  493. package/server/serverHelpers/serverUtilities.ts +104 -31
  494. package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
  495. package/server/serverHelpers/uwsServer.ts +17 -2
  496. package/server/static.ts +23 -29
  497. package/server/storageReclamation.ts +13 -0
  498. package/server/threads/itc.js +11 -1
  499. package/server/threads/manageThreads.js +526 -63
  500. package/server/threads/socketRouter.ts +81 -26
  501. package/server/threads/threadHeapMemory.ts +26 -0
  502. package/server/threads/threadServer.js +43 -15
  503. package/sqlTranslator/index.ts +31 -8
  504. package/studio/web/assets/{Chat-Br06zdMA.js → Chat-BnCBegQz.js} +1 -1
  505. package/studio/web/assets/{FloatingChat-BWImX5fA.js → FloatingChat-CoDW1ySS.js} +3 -3
  506. package/studio/web/assets/{apiToken-DN0nmDsq.js → apiToken-Bwk5BLXW.js} +1 -1
  507. package/studio/web/assets/{applications-kSxVoyeU.js → applications-DHxGi7JH.js} +1 -1
  508. package/studio/web/assets/{cssMode-C1JeufH5.js → cssMode-s0cWI-_M.js} +1 -1
  509. package/studio/web/assets/{editor-qoo9CrGO.js → editor-DNcRHK54.js} +1 -1
  510. package/studio/web/assets/{html-Dt4IIy04.js → html-Bdssedlg.js} +1 -1
  511. package/studio/web/assets/{htmlMode-DXgKKr4C.js → htmlMode-CoDlJ3fw.js} +1 -1
  512. package/studio/web/assets/{index-6onkYFOG.js → index-D6sxmFLR.js} +5 -5
  513. package/studio/web/assets/{index.lazy-BrCFnpNJ.js → index.lazy-tmU5BS8s.js} +1 -1
  514. package/studio/web/assets/{javascript-DNCQGUBc.js → javascript-B8meVSTH.js} +1 -1
  515. package/studio/web/assets/{jsonMode-CR6HWruP.js → jsonMode-DpIPd35T.js} +1 -1
  516. package/studio/web/assets/{languageServices-BM4fI4rS.js → languageServices-C_5FMJzQ.js} +1 -1
  517. package/studio/web/assets/{lspLanguageFeatures-DSa1ttcD.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
  518. package/studio/web/assets/{notifications-BHXLnh6x.js → notifications-CQf18QKb.js} +1 -1
  519. package/studio/web/assets/{notifications-CUmtIA6z.js → notifications-CvZivSbh.js} +1 -1
  520. package/studio/web/assets/{profile-8BeFSF3j.js → profile-DdOwtntb.js} +1 -1
  521. package/studio/web/assets/{regions-C8qR0HhD.js → regions-n69fwagr.js} +1 -1
  522. package/studio/web/assets/{register-B4n5i0SD.js → register-PfWTCXWB.js} +2 -2
  523. package/studio/web/assets/{setComponentFile-g0_B0lgX.js → setComponentFile-Bg6O7X0S.js} +1 -1
  524. package/studio/web/assets/{setup-B0CTj_Q5.js → setup-CUx_aUDl.js} +2 -2
  525. package/studio/web/assets/{status-C6Yib7-K.js → status-D7BVKqX9.js} +1 -1
  526. package/studio/web/assets/{toggleHighContrast-Dgta7bVi.js → toggleHighContrast-DBSyXzMr.js} +1 -1
  527. package/studio/web/assets/{tsMode-CH_jHvU-.js → tsMode-BByKCjBS.js} +1 -1
  528. package/studio/web/assets/{typescript-Co9LCXd5.js → typescript-DDLnLpw9.js} +1 -1
  529. package/studio/web/assets/{useEntityRestURL-wKC8NsC_.js → useEntityRestURL-31CHGaHk.js} +1 -1
  530. package/studio/web/assets/{workers-CWeLxCXA.js → workers-pR3jRY9D.js} +1 -1
  531. package/studio/web/assets/{xml-BadC-0Rk.js → xml-2iRnMhQO.js} +1 -1
  532. package/studio/web/assets/{yaml-BiUfxPbC.js → yaml-Bf92gJpd.js} +1 -1
  533. package/studio/web/index.html +1 -1
  534. package/upgrade/directives/5-3-0.ts +132 -0
  535. package/upgrade/directives/directivesController.ts +2 -1
  536. package/utility/OperationFunctionCaller.ts +2 -1
  537. package/utility/common_utils.ts +30 -5
  538. package/utility/componentNames.ts +12 -0
  539. package/utility/environment/environmentManager.ts +3 -7
  540. package/utility/environment/systemInformation.ts +7 -0
  541. package/utility/errors/commonErrors.ts +4 -0
  542. package/utility/errors/hdbError.ts +29 -0
  543. package/utility/hdbTerms.ts +17 -0
  544. package/utility/logging/harper_logger.ts +87 -14
  545. package/utility/logging/logRotator.ts +72 -45
  546. package/utility/nodeIdentity.ts +45 -0
  547. package/utility/npmUtilities.ts +10 -8
  548. package/utility/operationPermissions.ts +18 -1
  549. package/utility/operation_authorization.ts +231 -42
  550. package/utility/watchPath.ts +63 -0
  551. package/utility/watcherFallback.ts +270 -0
  552. package/validation/configValidator.ts +209 -72
  553. package/validation/installValidator.ts +15 -0
  554. package/validation/validationWrapper.ts +18 -4
@@ -41,7 +41,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
41
41
  return (mod && mod.__esModule) ? mod : { "default": mod };
42
42
  };
43
43
  Object.defineProperty(exports, "__esModule", { value: true });
44
- exports.EVICTED = exports.INVALIDATED = void 0;
44
+ exports.EVICTED = exports.INVALIDATED = exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT = void 0;
45
+ exports.acquireUpdateAttributesLock = acquireUpdateAttributesLock;
46
+ exports.releaseUpdateAttributesLock = releaseUpdateAttributesLock;
47
+ exports.withUpdateAttributesLock = withUpdateAttributesLock;
45
48
  exports.ignoreAlreadyDropped = ignoreAlreadyDropped;
46
49
  exports.freezeRecord = freezeRecord;
47
50
  exports.frozenRecordView = frozenRecordView;
@@ -50,6 +53,7 @@ exports.coerceType = coerceType;
50
53
  const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
51
54
  const node_vm_1 = require("node:vm");
52
55
  const node_crypto_1 = require("node:crypto");
56
+ const node_perf_hooks_1 = require("node:perf_hooks");
53
57
  const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
54
58
  const nodeIdMapping_ts_1 = require("./nodeIdMapping.js");
55
59
  const lodash_1 = __importDefault(require("lodash"));
@@ -58,6 +62,7 @@ const lmdbProcessRows_js_1 = __importDefault(require("../dataLayer/harperBridge/
58
62
  const Resource_ts_1 = require("./Resource.js");
59
63
  const when_ts_1 = require("../utility/when.js");
60
64
  const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
65
+ const recordLock_ts_1 = require("./recordLock.js");
61
66
  const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
62
67
  const transactionBroadcast_ts_1 = require("./transactionBroadcast.js");
63
68
  const hdbError_ts_1 = require("../utility/errors/hdbError.js");
@@ -93,7 +98,10 @@ const { validateAttribute } = lmdbProcessRows_js_1.default;
93
98
  const NULL_WITH_TIMESTAMP = new Uint8Array(9);
94
99
  NULL_WITH_TIMESTAMP[8] = 0xc0; // null
95
100
  const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
101
+ const MAX_DATE_TIMESTAMP = 8.64e15;
96
102
  const RECORD_PRUNING_INTERVAL = 60000; // one minute
103
+ const MAX_CONCURRENT_HISTORY_REMOVALS = 10;
104
+ const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
97
105
  // RocksDB-only: number of eviction/tombstone removals coalesced into a single transaction commit.
98
106
  // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
99
107
  // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
@@ -102,9 +110,75 @@ const EVICTION_BATCH_SIZE = 100;
102
110
  // letting an unbounded number of open transactions (and their snapshots) accumulate.
103
111
  const MAX_INFLIGHT_EVICTION_BATCHES = 4;
104
112
  const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
113
+ // Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
114
+ // of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
115
+ // into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
116
+ // truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
117
+ // (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
118
+ const MAX_EXACT_COUNT_SCAN = 1_000_000;
119
+ const MAX_EXACT_COUNT_MS = 1_000;
120
+ // Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
121
+ // not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
122
+ // streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
123
+ const MAX_COUNT_PAGE = 10_000;
124
+ // How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
125
+ // check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
126
+ const COUNT_YIELD_INTERVAL = 2_048;
105
127
  envMngr.initSync();
106
128
  const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
107
129
  const LOCK_TIMEOUT = 10000;
130
+ // This bounds schema-lock acquisition; LOCK_TIMEOUT bounds in-flight record writes during a drop.
131
+ exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000;
132
+ const UPDATE_ATTRIBUTES_LOCK = 'update-attributes';
133
+ // Contention is otherwise only visible once it becomes a timeout (harper#2251).
134
+ exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = 1000;
135
+ // raw ASCII bytes are ordered-binary's encoding of the string, so this addresses the same native
136
+ // lock as string-keyed tryLock/unlock calls
137
+ const updateAttributesLockKey = Buffer.from(UPDATE_ATTRIBUTES_LOCK);
138
+ const lockWait = new Int32Array(new SharedArrayBuffer(4));
139
+ /** The wait blocks the event loop, so the locked section must stay synchronous. */
140
+ function acquireUpdateAttributesLock(rootStore, scopeDescription, timeout = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT) {
141
+ if (rootStore.tryLock(updateAttributesLockKey))
142
+ return;
143
+ const startTime = node_perf_hooks_1.performance.now();
144
+ let waitTime = 1;
145
+ while (!rootStore.tryLock(updateAttributesLockKey)) {
146
+ const elapsed = node_perf_hooks_1.performance.now() - startTime;
147
+ if (elapsed >= timeout) {
148
+ throw new hdbError_ts_1.UpdateAttributesLockTimeoutError(`Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`);
149
+ }
150
+ if (elapsed >= 2) {
151
+ Atomics.wait(lockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
152
+ if (waitTime < 16)
153
+ waitTime *= 2;
154
+ }
155
+ }
156
+ const waited = node_perf_hooks_1.performance.now() - startTime;
157
+ // The caller cannot register its release until we return, so a throw here would leak the lock
158
+ // with no `finally` able to reach it.
159
+ if (waited >= exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
160
+ try {
161
+ logger_ts_1.logger.warn?.(`Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms; this worker's event loop was blocked for that wait, and a holder that runs past ${exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT}ms fails the update outright`);
162
+ }
163
+ catch { }
164
+ }
165
+ function releaseUpdateAttributesLock(rootStore) {
166
+ rootStore.unlock(updateAttributesLockKey);
167
+ }
168
+ function withUpdateAttributesLock(rootStore, scopeDescription, callback) {
169
+ acquireUpdateAttributesLock(rootStore, scopeDescription);
170
+ try {
171
+ const result = callback();
172
+ if (typeof result?.then === 'function') {
173
+ Promise.resolve(result).catch((error) => logger_ts_1.logger.error?.(`Async update-attributes callback rejected after its lock was released (${scopeDescription})`, error));
174
+ throw new TypeError(`withUpdateAttributesLock callback must be synchronous (${scopeDescription}); asynchronous work may continue after the lock is released`);
175
+ }
176
+ return result;
177
+ }
178
+ finally {
179
+ releaseUpdateAttributesLock(rootStore);
180
+ }
181
+ }
108
182
  // Tolerate a redundant column family drop. Drops are broadcast to every worker
109
183
  // thread and each holds its own handle to the same underlying family, so a
110
184
  // concurrent worker may already have dropped it; the storage engine reports
@@ -335,8 +409,24 @@ function detectCyclicEnumerable(start) {
335
409
  function chainKeyForId(id) {
336
410
  return typeof id === 'string' ? 's' + id : 'k' + (0, DatabaseTransaction_ts_1.writeKeyId)(id);
337
411
  }
412
+ /** Normalizes a passed `context` argument as `transactional()` does; undefined means fall back to ambient. */
413
+ function contextArgument(context) {
414
+ if (!context || (0, DatabaseTransaction_ts_1.isReleasedTransaction)(context))
415
+ return undefined;
416
+ const resolved = context.getContext?.() || context;
417
+ return resolved instanceof DatabaseTransaction_ts_1.DatabaseTransaction ? { transaction: resolved } : resolved;
418
+ }
419
+ /** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
420
+ function isPlainOptions(value) {
421
+ return (typeof value === 'object' &&
422
+ value !== null &&
423
+ !Array.isArray(value) &&
424
+ !(value instanceof URLSearchParams) &&
425
+ value.id === undefined);
426
+ }
338
427
  function makeTable(options) {
339
- const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, } = options;
428
+ var _a;
429
+ const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, } = options;
340
430
  let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
341
431
  evictionMs ??= 0;
342
432
  // Eviction without explicit expiration means expiration:0. Apply at construction so
@@ -350,6 +440,7 @@ function makeTable(options) {
350
440
  properties = (0, jsonSchemaTypes_ts_1.projectAttributesToProperties)(attributes);
351
441
  const updateRecord = (0, RecordEncoder_ts_1.recordUpdater)(primaryStore, tableId, auditStore);
352
442
  let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
443
+ let warnedFutureSourceVersion = false; // likewise (see getFromSource)
353
444
  let sourceLoad; // if a source has a load function (replicator), record it here
354
445
  let hasSourceGet;
355
446
  let primaryKeyAttribute;
@@ -383,6 +474,10 @@ function makeTable(options) {
383
474
  let cleanupPriority = 0;
384
475
  let lastCleanupInterval;
385
476
  let cleanupTimer;
477
+ let recordExpirationInterval;
478
+ // a reclamation pass awaits a scheduled cleanup, which only settles from its timer
479
+ const pendingCleanupResolvers = new Set();
480
+ let disposed = false;
386
481
  // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
387
482
  let expirationScanScheduled = false;
388
483
  // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
@@ -431,10 +526,11 @@ function makeTable(options) {
431
526
  const MAX_PREFETCH_BUNDLE = 6;
432
527
  if (audit)
433
528
  addDeleteRemoval();
434
- (0, storageReclamation_ts_1.onStorageReclamation)(primaryStore.path, (priority) => {
529
+ const reclamationHandler = (priority) => {
435
530
  if (hasSourceGet)
436
531
  return scheduleCleanup(priority);
437
- });
532
+ };
533
+ (0, storageReclamation_ts_1.onStorageReclamation)(primaryStore.path, reclamationHandler);
438
534
  class Updatable extends tracked_ts_1.GenericTrackedObject {
439
535
  getUpdatedTime() {
440
536
  return RecordEncoder_ts_1.entryMap.get(this.getRecord())?.version;
@@ -533,12 +629,68 @@ function makeTable(options) {
533
629
  },
534
630
  });
535
631
  }
632
+ function resolveAuditHead(id, version, nodeId, refs) {
633
+ if (!refs?.length)
634
+ return { txnLogKey: version, nodeId };
635
+ const visited = new Set();
636
+ function findHead(candidateRefs) {
637
+ if (!candidateRefs)
638
+ return;
639
+ const pending = candidateRefs.slice().reverse();
640
+ while (pending.length > 0) {
641
+ const ref = pending.pop();
642
+ const identity = `${ref.nodeId ?? 0}:${ref.version}`;
643
+ if (visited.has(identity))
644
+ continue;
645
+ visited.add(identity);
646
+ const entry = auditStore.getSync(ref.version, tableId, id, ref.nodeId);
647
+ if (!entry)
648
+ continue;
649
+ if (entry.version === version && (nodeId == null || (entry.nodeId ?? 0) === nodeId))
650
+ return { txnLogKey: ref.version, nodeId: ref.nodeId };
651
+ const previousRefs = entry.previousAdditionalAuditRefs;
652
+ if (previousRefs) {
653
+ for (let index = previousRefs.length - 1; index >= 0; index--)
654
+ pending.push(previousRefs[index]);
655
+ }
656
+ }
657
+ }
658
+ const referencedHead = findHead(refs);
659
+ if (referencedHead)
660
+ return referencedHead;
661
+ if (version != null) {
662
+ const directHead = auditStore.getSync(version, tableId, id, nodeId);
663
+ if (directHead?.version === version && (nodeId == null || (directHead.nodeId ?? 0) === nodeId))
664
+ return { txnLogKey: version, nodeId };
665
+ }
666
+ return { txnLogKey: version, nodeId };
667
+ }
536
668
  class TableResource extends Resource_ts_1.Resource {
537
669
  #record; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
538
670
  #changes; // the changes to the record that have been made (should not be modified directly)
539
671
  #version; // version of the record
540
672
  #entry; // the entry from the database
541
673
  #savingOperation; // operation for the record is currently being saved
674
+ #lockHandle; // the record lock acquired by lock() — scoped or hold
675
+ #lockWritable; // set by #reloadLocked to let save() stage lock-writable updates
676
+ /**
677
+ * Shared guard: if this instance is lock-writable but the handle is gone (expired or
678
+ * released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
679
+ * in addition to the save() path. Every lock-writable instance carries its own handle in
680
+ * #lockHandle (scoped and hold alike), so we never need to search the registry here.
681
+ */
682
+ #assertLiveHandle(id) {
683
+ if (!this.#lockWritable)
684
+ return;
685
+ const handle = this.#lockHandle;
686
+ // Off-key writes through the same resource instance are ordinary; only guard the
687
+ // exact key the lock was acquired for.
688
+ if (handle.keyId !== (0, DatabaseTransaction_ts_1.writeKeyId)(id))
689
+ return;
690
+ if (handle.expired || handle.released) {
691
+ throw (0, recordLock_ts_1.lockNotHeldError)(handle);
692
+ }
693
+ }
542
694
  // #section: static-config
543
695
  static name = tableName; // for display/debugging purposes
544
696
  static primaryStore = primaryStore;
@@ -635,7 +787,7 @@ function makeTable(options) {
635
787
  // perform the write of an individual write event
636
788
  const writeUpdate = async (event, context) => {
637
789
  const value = event.value;
638
- const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : TableResource;
790
+ const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
639
791
  if (databaseName === hdbTerms_ts_1.SYSTEM_SCHEMA_NAME &&
640
792
  (event.table === hdbTerms_ts_1.SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME || event.table === hdbTerms_ts_1.SYSTEM_TABLE_NAMES.USER_TABLE_NAME)) {
641
793
  userRoleUpdate = true;
@@ -652,6 +804,9 @@ function makeTable(options) {
652
804
  ensureLoaded: false,
653
805
  nodeId: event.nodeId,
654
806
  viaNodeId: event.viaNodeId,
807
+ // the origin's record version, stored as-is so every replica holds the version the
808
+ // origin holds; the transaction's own timestamp stays the origin's log key
809
+ version: event.version,
655
810
  // use per-event expiresAt: batched txn context only holds the first event's expiration
656
811
  expiresAt: event.expiresAt,
657
812
  // bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log
@@ -690,7 +845,7 @@ function makeTable(options) {
690
845
  const stageWrite = (event, context) => {
691
846
  let chainKey;
692
847
  try {
693
- const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : TableResource;
848
+ const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
694
849
  const id = event.id ?? (event.value ? event.value[Table?.primaryKey] : undefined);
695
850
  if (id != null && typeof id !== 'symbol')
696
851
  chainKey = `${event.table ?? tableName} ${chainKeyForId(id)}`;
@@ -890,7 +1045,9 @@ function makeTable(options) {
890
1045
  continue;
891
1046
  }
892
1047
  }
893
- // use the version as the transaction timestamp
1048
+ // A source that reports no log position of its own (no `timestamp`) has only one clock,
1049
+ // so its record version doubles as the apply transaction's timestamp. A replication
1050
+ // receiver always sets `timestamp` from the origin's log key and never reaches this.
894
1051
  if (!event.timestamp && event.version)
895
1052
  event.timestamp = event.version;
896
1053
  const commitResolution = (0, transaction_ts_1.transaction)(event, () => {
@@ -991,7 +1148,7 @@ function makeTable(options) {
991
1148
  /** Indicates if the events should be revalidated when they are received. By default we do this if the get
992
1149
  * method is overriden */
993
1150
  static get shouldRevalidateEvents() {
994
- return this.prototype.get !== TableResource.prototype.get;
1151
+ return this.prototype.get !== _a.prototype.get;
995
1152
  }
996
1153
  /**
997
1154
  * Gets a resource instance, as defined by the Resource class, adding the table-specific handling
@@ -1027,7 +1184,7 @@ function makeTable(options) {
1027
1184
  }
1028
1185
  return loadLocalRecord(id, request, { transaction: readTxn, ensureLoaded: resourceOptions?.ensureLoaded }, sync, (entry) => {
1029
1186
  if (entry) {
1030
- TableResource._updateResource(this, entry);
1187
+ _a._updateResource(this, entry);
1031
1188
  }
1032
1189
  else
1033
1190
  this.#record = null;
@@ -1045,7 +1202,7 @@ function makeTable(options) {
1045
1202
  if (loadingFromSource) {
1046
1203
  txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
1047
1204
  return (0, when_ts_1.when)(loadingFromSource, (entry) => {
1048
- TableResource._updateResource(this, entry);
1205
+ _a._updateResource(this, entry);
1049
1206
  return this;
1050
1207
  });
1051
1208
  }
@@ -1265,7 +1422,7 @@ function makeTable(options) {
1265
1422
  return dbisDb.getSync([Symbol.for('residency_by_id'), id]);
1266
1423
  }
1267
1424
  static setResidency(getResidency) {
1268
- TableResource.getResidency =
1425
+ _a.getResidency =
1269
1426
  getResidency &&
1270
1427
  ((record, context) => {
1271
1428
  try {
@@ -1278,7 +1435,7 @@ function makeTable(options) {
1278
1435
  });
1279
1436
  }
1280
1437
  static setResidencyById(getResidencyById) {
1281
- TableResource.getResidencyById =
1438
+ _a.getResidencyById =
1282
1439
  getResidencyById &&
1283
1440
  ((id) => {
1284
1441
  try {
@@ -1291,8 +1448,8 @@ function makeTable(options) {
1291
1448
  });
1292
1449
  }
1293
1450
  static getResidency(record, context) {
1294
- if (TableResource.getResidencyById) {
1295
- return TableResource.getResidencyById(record[primaryKey]);
1451
+ if (_a.getResidencyById) {
1452
+ return _a.getResidencyById(record[primaryKey]);
1296
1453
  }
1297
1454
  let count = replicateToCount;
1298
1455
  if (context.replicateTo != undefined) {
@@ -1334,7 +1491,7 @@ function makeTable(options) {
1334
1491
  return; // already enabled
1335
1492
  audit = true;
1336
1493
  addDeleteRemoval();
1337
- TableResource.audit = true;
1494
+ _a.audit = true;
1338
1495
  }
1339
1496
  /**
1340
1497
  * Coerce the id as a string to the correct type for the primary key
@@ -1346,7 +1503,24 @@ function makeTable(options) {
1346
1503
  return null;
1347
1504
  return coerceType(id, primaryKeyAttribute);
1348
1505
  }
1506
+ /**
1507
+ * A branch's Table classes deliberately carry the BASE's logical database name so an
1508
+ * application's schema and code resolve unchanged (harper#643). That makes every schema
1509
+ * mutation resolve against the global catalog — a `dropTable()` through a branch would delete
1510
+ * the live base table. Reads and writes are per-branch and unaffected; DDL is refused until a
1511
+ * branch owns a schema identity of its own.
1512
+ */
1513
+ static assertSchemaMutable(operation) {
1514
+ if (!isBranch)
1515
+ return;
1516
+ const error = new Error(`Cannot ${operation} through a branched database: '${tableName}' resolves to the schema of base ` +
1517
+ `database '${databaseName}', so the change would apply to the base rather than the branch`);
1518
+ error.statusCode = 400;
1519
+ throw error;
1520
+ }
1349
1521
  static async dropTable() {
1522
+ _a.assertSchemaMutable('drop a table');
1523
+ const rootStore = primaryStore.rootStore;
1350
1524
  if (databaseName === databasePath) {
1351
1525
  // Persist a drop tombstone on the primary catalog entry BEFORE any
1352
1526
  // destructive work. If the process dies or a column family drop fails
@@ -1354,9 +1528,11 @@ function makeTable(options) {
1354
1528
  // the next startup (or a same-name create) completes the drop via
1355
1529
  // completeInterruptedDrop in databases.ts instead of resurrecting
1356
1530
  // the table.
1357
- const primaryCatalogKey = TableResource.tableName + '/';
1358
- const primaryMeta = dbisDb.getSync(primaryCatalogKey);
1359
- if (primaryMeta && !primaryMeta.dropping) {
1531
+ const primaryCatalogKey = _a.tableName + '/';
1532
+ const writeTombstone = () => {
1533
+ const primaryMeta = dbisDb.getSync(primaryCatalogKey);
1534
+ if (!primaryMeta || primaryMeta.dropping)
1535
+ return;
1360
1536
  primaryMeta.dropping = true;
1361
1537
  // Stamps this drop's identity so the interrupted-drop retry budget in
1362
1538
  // databases.ts can be scoped to THIS drop rather than the table name: a
@@ -1366,11 +1542,23 @@ function makeTable(options) {
1366
1542
  // the budget by generation instead makes the new drop's tombstone carry
1367
1543
  // its own fresh key regardless of what any worker last observed.
1368
1544
  primaryMeta.dropGeneration = (0, node_crypto_1.randomUUID)();
1369
- // put is rebound to putSync on RocksDB stores; on LMDB it returns
1370
- // a promise, so await it to make the tombstone durable before the
1371
- // destructive work below
1372
- const tombstoneWrite = dbisDb.put(primaryCatalogKey, primaryMeta);
1373
- if (tombstoneWrite?.then)
1545
+ return dbisDb.put(primaryCatalogKey, primaryMeta);
1546
+ };
1547
+ if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
1548
+ // withUpdateAttributesLock's locked section cannot be held across an await, so a durable
1549
+ // tombstone depends on put being rebound to putSync for RocksDB primary stores (see
1550
+ // createOpenDBIObject). Check that BEFORE writing anything: a tombstone left behind by a
1551
+ // refused drop would delete the table on the next load.
1552
+ if (dbisDb.put !== dbisDb.putSync)
1553
+ throw new Error(`Cannot drop ${databaseName}.${_a.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`);
1554
+ withUpdateAttributesLock(rootStore, `drop table '${databaseName}.${_a.tableName}'`, writeTombstone);
1555
+ }
1556
+ else {
1557
+ let tombstoneWrite;
1558
+ rootStore.transactionSync(() => {
1559
+ tombstoneWrite = writeTombstone();
1560
+ });
1561
+ if (typeof tombstoneWrite?.then === 'function')
1374
1562
  await tombstoneWrite;
1375
1563
  }
1376
1564
  }
@@ -1441,28 +1629,25 @@ function makeTable(options) {
1441
1629
  // same-name create completes the interrupted drop and writes fresh
1442
1630
  // catalog rows, and clobbering those would orphan the new table.
1443
1631
  const removeTombstonedCatalog = () => {
1444
- const currentPrimary = dbisDb.getSync(TableResource.tableName + '/');
1632
+ const currentPrimary = dbisDb.getSync(_a.tableName + '/');
1445
1633
  if (!currentPrimary?.dropping)
1446
1634
  return false;
1447
1635
  for (const attribute of attributes) {
1448
- dbisDb.remove(TableResource.tableName + '/' + attribute.name);
1636
+ dbisDb.remove(_a.tableName + '/' + attribute.name);
1449
1637
  }
1450
- dbisDb.remove(TableResource.tableName + '/');
1638
+ dbisDb.remove(_a.tableName + '/');
1451
1639
  return true;
1452
1640
  };
1453
- const rootStore = primaryStore.rootStore;
1454
1641
  if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
1455
1642
  // Serialize the drops + catalog removal against a concurrent
1456
1643
  // same-name create (and completeInterruptedDrop) under the database's
1457
1644
  // 'update-attributes' exclusive lock - the same lock the create path
1458
- // holds. It is a synchronous spin lock that blocks the event loop, so
1645
+ // holds. It is a synchronous lock wait that blocks the event loop, so
1459
1646
  // the locked section MUST stay synchronous: drop with dropSync (as
1460
1647
  // completeInterruptedDrop does), never an awaited drop(), or a
1461
- // concurrent create's spin would deadlock waiting on a drop that the
1462
- // blocked event loop can never resolve.
1463
- while (!rootStore.tryLock('update-attributes')) { }
1464
- let removed = false;
1465
- try {
1648
+ // concurrent create's wait would be stuck on a drop that the blocked
1649
+ // event loop can never resolve, burning its full deadline before failing.
1650
+ const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
1466
1651
  for (const attribute of attributes) {
1467
1652
  const index = indices[attribute.name];
1468
1653
  if (index)
@@ -1479,11 +1664,8 @@ function makeTable(options) {
1479
1664
  catch (error) {
1480
1665
  ignoreAlreadyDropped(error);
1481
1666
  }
1482
- removed = removeTombstonedCatalog();
1483
- }
1484
- finally {
1485
- rootStore.unlock('update-attributes');
1486
- }
1667
+ return removeTombstonedCatalog();
1668
+ });
1487
1669
  if (removed)
1488
1670
  await dbisDb.committed;
1489
1671
  }
@@ -1540,7 +1722,7 @@ function makeTable(options) {
1540
1722
  estimatedRecordRange: undefined,
1541
1723
  };
1542
1724
  if (this.getContext()?.includeExpensiveRecordCountEstimates) {
1543
- return TableResource.getRecordCount().then((recordCount) => {
1725
+ return _a.getRecordCount().then((recordCount) => {
1544
1726
  description.recordCount = recordCount.recordCount;
1545
1727
  description.estimatedRecordRange = recordCount.estimatedRange;
1546
1728
  return description;
@@ -1824,14 +2006,71 @@ function makeTable(options) {
1824
2006
  * Save any changes into this instance to the current transaction
1825
2007
  */
1826
2008
  save() {
1827
- if (this.#savingOperation) {
1828
- try {
1829
- return this.#saveOperation(this.#savingOperation);
2009
+ this.#assertLiveHandle(this.getId()); // a write through a released or expired lock never lands
2010
+ const operation = this.#savingOperation;
2011
+ if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
2012
+ // A held lock's record stages its update here rather than at lock() time: it is often
2013
+ // written after the acquiring transaction has already completed, which would have
2014
+ // dropped an update staged then. Nothing set means nothing to stage — a held-but-untouched
2015
+ // id stays untouched. Scoped locks do not take this branch: #reloadLocked stages their
2016
+ // TransactionWrite at lock() time (exactly like update()), so #savingOperation is always
2017
+ // set for a live scoped lock and the ordinary path below applies.
2018
+ // Verify the hold is still alive: if #lockWritable is set but the handle expired or was
2019
+ // released between lock acquisition and this save(), throw 409 rather than silently
2020
+ // committing stale data. Every lock-writable instance carries its own handle.
2021
+ const saveHandle = this.#lockHandle;
2022
+ if (saveHandle.expired || saveHandle.released) {
2023
+ throw (0, recordLock_ts_1.lockNotHeldError)(saveHandle);
2024
+ }
2025
+ const changes = this.#changes;
2026
+ if (changes && Object.keys(changes).length > 0) {
2027
+ this.#savingOperation = null;
2028
+ return (0, when_ts_1.when)(this._writeUpdate(this.getId(), changes, false), () => {
2029
+ const op = this.#savingOperation;
2030
+ if (op?.dropped) {
2031
+ this.#changes = undefined;
2032
+ return;
2033
+ }
2034
+ // Clear #savingOperation so the next sequential save() enters the lock-writable
2035
+ // path and creates a fresh write (otherwise a non-null #savingOperation makes
2036
+ // save() take the #saveOperation branch with an already-committed write, which
2037
+ // is a no-op, silently dropping the new change).
2038
+ // op.innerCommit is the real native-transaction commit Promise set on the
2039
+ // immediateCommit path in DatabaseTransaction.save(); await it to ensure
2040
+ // durability before resolving to the caller.
2041
+ if (op?.saved) {
2042
+ this.#savingOperation = null;
2043
+ return op?.innerCommit;
2044
+ }
2045
+ // op.saved = false means addWrite deferred the save; #saveOperation commits it
2046
+ // synchronously but ImmediateTransaction.save() returns undefined while the
2047
+ // inner rocksdb commit is still pending — return innerCommit so the caller
2048
+ // actually waits for durability.
2049
+ return (0, when_ts_1.when)(this.save(), () => op?.innerCommit);
2050
+ });
1830
2051
  }
1831
- finally {
2052
+ // No changes: nothing to stage. A dropped operation (detached at a scoped→hold
2053
+ // upgrade — see detachScopedUpgradeWrite) must not fall through to the ordinary
2054
+ // #saveOperation path below with its now-detached reference.
2055
+ if (!operation || operation.dropped) {
1832
2056
  this.#savingOperation = null;
2057
+ return;
1833
2058
  }
1834
2059
  }
2060
+ if (this.#savingOperation) {
2061
+ const operation = this.#savingOperation;
2062
+ this.#savingOperation = null;
2063
+ // A write that lands via a nested immediateCommit (e.g. a second sequential save() on
2064
+ // the same ImmediateTransaction context, once the first has already closed it) sets
2065
+ // operation.innerCommit to the real native-commit promise, but the commit() sweep loop
2066
+ // that triggers it discards its own return value — #saveOperation()'s result can
2067
+ // resolve before that native commit actually settles. Chain on innerCommit (as the
2068
+ // lock-writable hold branch above already does) so callers awaiting save() see the
2069
+ // write durably land, not just the outer (possibly premature) resolution.
2070
+ const result = this.#saveOperation(operation);
2071
+ const innerCommit = operation.innerCommit;
2072
+ return innerCommit ? (0, when_ts_1.when)(innerCommit, () => result) : result;
2073
+ }
1835
2074
  }
1836
2075
  #saveOperation(operation) {
1837
2076
  const transaction = txnForContext(this.getContext());
@@ -1909,6 +2148,7 @@ function makeTable(options) {
1909
2148
  });
1910
2149
  }
1911
2150
  _writeInvalidate(id, partialRecord, options) {
2151
+ this.#assertLiveHandle(id);
1912
2152
  const context = this.getContext();
1913
2153
  checkValidId(id);
1914
2154
  const transaction = txnForContext(this.getContext());
@@ -1917,9 +2157,12 @@ function makeTable(options) {
1917
2157
  store: primaryStore,
1918
2158
  invalidated: true,
1919
2159
  entry: this.#entry,
2160
+ recordVersion: options?.version,
2161
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
1920
2162
  commit: (txnTime, existingEntry, _retry, transaction) => {
2163
+ const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
1921
2164
  write.skipped = false; // reset on each retry; cleanup happens after commit if still true
1922
- if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) {
2165
+ if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
1923
2166
  write.skipped = true;
1924
2167
  return;
1925
2168
  }
@@ -1940,7 +2183,13 @@ function makeTable(options) {
1940
2183
  viaNodeId: options?.viaNodeId,
1941
2184
  transaction,
1942
2185
  tableToTrack: tableName,
2186
+ recordVersion: txnTime,
2187
+ additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
2188
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
2189
+ : undefined,
1943
2190
  }, 'invalidate');
2191
+ if (write.trackRecordVersion)
2192
+ write.recordVersionApplied = true;
1944
2193
  // TODO: recordDeletion?
1945
2194
  },
1946
2195
  };
@@ -1948,21 +2197,25 @@ function makeTable(options) {
1948
2197
  transaction.addWrite(write);
1949
2198
  }
1950
2199
  _writeRelocate(id, options) {
2200
+ this.#assertLiveHandle(id);
1951
2201
  const context = this.getContext();
1952
2202
  checkValidId(id);
1953
2203
  const transaction = txnForContext(this.getContext());
1954
- transaction.addWrite({
2204
+ const write = {
1955
2205
  key: id,
1956
2206
  store: primaryStore,
1957
2207
  invalidated: true,
1958
2208
  entry: this.#entry,
2209
+ recordVersion: options?.version,
2210
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
1959
2211
  before: this.constructor.source?.relocate && !context?.source
1960
2212
  ? this.constructor.source.relocate.bind(this.constructor.source, id, undefined, context)
1961
2213
  : undefined,
1962
2214
  commit: (txnTime, existingEntry, _retry, transaction) => {
1963
- if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0)
2215
+ const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2216
+ if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0)
1964
2217
  return;
1965
- const residency = TableResource.getResidencyRecord(options.residencyId);
2218
+ const residency = _a.getResidencyRecord(options.residencyId);
1966
2219
  let metadata = 0;
1967
2220
  let newRecord = null;
1968
2221
  const existingRecord = existingEntry?.value;
@@ -1986,9 +2239,16 @@ function makeTable(options) {
1986
2239
  viaNodeId: options?.viaNodeId,
1987
2240
  expiresAt: options.expiresAt,
1988
2241
  transaction,
2242
+ recordVersion: txnTime,
2243
+ additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
2244
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
2245
+ : undefined,
1989
2246
  }, 'relocate', false, null);
2247
+ if (write.trackRecordVersion)
2248
+ write.recordVersionApplied = true;
1990
2249
  },
1991
- });
2250
+ };
2251
+ transaction.addWrite(write);
1992
2252
  }
1993
2253
  /**
1994
2254
  * Record the relocation of an entry (when a record is moved to a different node), return true if it is now located locally
@@ -2107,10 +2367,253 @@ function makeTable(options) {
2107
2367
  }
2108
2368
  }
2109
2369
  /**
2110
- * This is intended to acquire a lock on a record from the whole cluster.
2370
+ * Static entry point: `Table.lock(id, options?, context?)` — creates an instance in the given,
2371
+ * ambient, or a fresh context and delegates to the instance lock(). This shadows Resource.static
2372
+ * lock so that both callers share the same transaction link (required for cross-instance upgrade
2373
+ * detection). lock() is an in-process API with no authorization hook of its own; it is not
2374
+ * protocol-dispatched, so no allowUpdate/allowCreate check runs on acquisition.
2375
+ *
2376
+ * Dropping the trailing `context` leaks the key: the bare `{}` fallback is an
2377
+ * ImmediateTransaction, which releases no record locks.
2378
+ */
2379
+ static async lock(target, options, context) {
2380
+ if (!isRocksDB)
2381
+ throw new hdbError_ts_1.ClientError('Record locks are not supported on LMDB', 501);
2382
+ if (options === undefined && isPlainOptions(target)) {
2383
+ options = target;
2384
+ target = undefined;
2385
+ }
2386
+ const id = target != null ? requestTargetToId(target) : null;
2387
+ const resolvedContext = contextArgument(context) ?? transaction_ts_1.contextStorage.getStore() ?? {};
2388
+ const resource = new _a(id, resolvedContext);
2389
+ return resource.lock(target, options);
2390
+ }
2391
+ /**
2392
+ * Acquire an exclusive lock on this record (or on `target`'s) and return it ready for updates
2393
+ * (harper#483, Phase 0: exclusive across every worker thread of this node). The lock is held
2394
+ * in process memory only — no durable writes. Phase 0 contract: lock() is mutually exclusive
2395
+ * with other lock() calls on the same key; plain writes (put/patch/delete/create) are never
2396
+ * gated or blocked. The generation expires after `lease` if it is never released.
2397
+ *
2398
+ * Transaction-scoped (default): write through the returned record (or the table's static verbs
2399
+ * in the same transaction), and the commit or abort releases it. `{ hold: true }`: the lock
2400
+ * outlives the transaction; write through the returned record and release with `unlock()`, or
2401
+ * let the lease expire.
2402
+ */
2403
+ // async so option/id validation rejects rather than throwing past a caller's `.catch()`; the
2404
+ // body still runs to completion synchronously, which is what keeps concurrent lock() calls
2405
+ // on one key coalescing instead of racing to tryLock.
2406
+ async lock(target, options) {
2407
+ if (!isRocksDB)
2408
+ throw new hdbError_ts_1.ClientError('Record locks are not supported on LMDB', 501);
2409
+ if (options === undefined && isPlainOptions(target)) {
2410
+ options = target;
2411
+ target = undefined;
2412
+ }
2413
+ const id = target != null ? requestTargetToId(target) : this.getId();
2414
+ checkValidId(id);
2415
+ const resolved = (0, recordLock_ts_1.resolveLockOptions)(options);
2416
+ const context = this.getContext();
2417
+ const link = txnForContext(context);
2418
+ const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
2419
+ const held = this.#lockHandle;
2420
+ if (held && !held.released && !held.expired && held.keyId === keyId) {
2421
+ // Re-entrant: upgrade to hold if requested, then preserve staged changes.
2422
+ if (resolved.hold && !held.hold) {
2423
+ held.upgradeToHold(resolved.lease);
2424
+ // The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
2425
+ // staging is deferred and explicit-save-only, so an unsaved scoped write left
2426
+ // dangling here would otherwise auto-commit at the transaction sweep and clobber
2427
+ // whatever the hold write lands. detachScopedUpgradeWrite marks it .dropped so a
2428
+ // later save() on this instance falls through to the hold branch instead of the
2429
+ // dead #savingOperation reference.
2430
+ detachScopedUpgradeWrite(link, keyId, held);
2431
+ }
2432
+ return Promise.resolve(this.#reloadLocked(id, undefined, true));
2433
+ }
2434
+ const scoped = link.recordLockFor(primaryStore, keyId);
2435
+ if (scoped && !scoped.released && !scoped.expired) {
2436
+ if (resolved.hold && !scoped.hold) {
2437
+ // Upgrade scoped → hold: flip the existing handle object to hold mode so every
2438
+ // instance that already references this handle stays valid. Retiring and creating a
2439
+ // new handle would invalidate those other references (their save() would then throw
2440
+ // 409 against a released handle). The native key stays locked throughout.
2441
+ scoped.upgradeToHold(resolved.lease);
2442
+ detachScopedUpgradeWrite(link, keyId, scoped);
2443
+ return Promise.resolve(this.#reloadLocked(id, scoped, true));
2444
+ }
2445
+ // Already held with the same type: re-entrant return. Preserve any staged changes.
2446
+ return Promise.resolve(this.#reloadLocked(id, scoped, true));
2447
+ }
2448
+ const key = (0, recordLock_ts_1.lockAttemptKey)(tableId, id);
2449
+ // Coalesce concurrent lock() calls for the same key inside one link so they don't
2450
+ // self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
2451
+ // reach tryLock before either registers, making the second park against the first.
2452
+ const pending = link.pendingLockFor(primaryStore, keyId);
2453
+ if (pending) {
2454
+ // Wait for the in-flight acquisition, then take the re-entrant path as if
2455
+ // recordLockFor had found it. If the first attempt timed out, re-enter so
2456
+ // the second caller gets its own timeout.
2457
+ // The follower waits on the leader's acquisition, but only for its own timeout.
2458
+ let followerTimer;
2459
+ const followerTimedOut = Symbol('follower timeout');
2460
+ const followerStart = Date.now();
2461
+ const followerDeadline = new Promise((_, reject) => {
2462
+ followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
2463
+ });
2464
+ // Try again on this caller's own terms with the budget it has left.
2465
+ const retryOnRemainingBudget = () => {
2466
+ // The enclosing transaction ended while we were parked. A retry re-resolves the
2467
+ // context, which no longer points at this link, so the handle it acquired would be
2468
+ // registered on a fresh transaction that no commit or abort ever releases — the
2469
+ // same abandonment the leader's own post-acquisition guard below rejects.
2470
+ if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits)
2471
+ throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
2472
+ const remaining = resolved.timeout - (Date.now() - followerStart);
2473
+ if (remaining <= 0)
2474
+ throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
2475
+ return this.lock(target, { ...resolved, timeout: remaining });
2476
+ };
2477
+ return Promise.race([pending, followerDeadline]).then(() => {
2478
+ clearTimeout(followerTimer);
2479
+ const acquired = link.recordLockFor(primaryStore, keyId);
2480
+ if (acquired && !acquired.released && !acquired.expired) {
2481
+ if (resolved.hold && !acquired.hold) {
2482
+ detachScopedUpgradeWrite(link, keyId, acquired);
2483
+ acquired.upgradeToHold(resolved.lease);
2484
+ }
2485
+ return this.#reloadLocked(id, acquired, true);
2486
+ }
2487
+ return retryOnRemainingBudget();
2488
+ }, (error) => {
2489
+ clearTimeout(followerTimer);
2490
+ if (error === followerTimedOut)
2491
+ throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
2492
+ return retryOnRemainingBudget();
2493
+ });
2494
+ }
2495
+ const pendingPromise = (0, recordLock_ts_1.acquireRecordKey)(link, primaryStore, key, keyId, resolved.timeout, resolved.lease, resolved.hold);
2496
+ link.registerPendingLock(primaryStore, keyId, pendingPromise);
2497
+ return pendingPromise.then((handle) => {
2498
+ link.unregisterPendingLock(primaryStore, keyId);
2499
+ if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits) {
2500
+ // The transaction was aborted while this call waited; nothing would ever release the handle.
2501
+ handle.release();
2502
+ throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
2503
+ }
2504
+ link.registerRecordLock(handle);
2505
+ if (link.saveCommits && context?.timestamp)
2506
+ handle.noteCandidateFloor(context.timestamp);
2507
+ if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.OPEN && !link.saveCommits) {
2508
+ // Explicit transaction() (not ImmediateTransaction): pin the clock to
2509
+ // acquiredAt when no writes have been staged yet. When writes already
2510
+ // exist, leave the clock alone (ordering is best-effort; write held records
2511
+ // in their own transaction for the guarantee). ImmediateTransaction is
2512
+ // excluded (saveCommits=true) — its clock is never pinned in lock();
2513
+ // each save() stamps from the handle's committed version floor instead.
2514
+ if (link.writes.length === 0 && !link.timestamp) {
2515
+ link.timestamp = handle.acquiredAt;
2516
+ }
2517
+ if (!resolved.hold && link.transaction) {
2518
+ // Scoped lock: the read snapshot may predate the lock; drop it so the
2519
+ // scope reads what it locked. Hold locks use acquiredAt directly and
2520
+ // do not update the read snapshot.
2521
+ // The timestamp guard matches DatabaseTransaction's own setTimestamp calls: a
2522
+ // deferred update() write leaves the clock at 0, which rocksdb-js rejects.
2523
+ if (link.writes.length === 0 && link.readTxnsUsed <= 1) {
2524
+ link.releaseReadTxn();
2525
+ link.snapshotFree = true;
2526
+ }
2527
+ else if (link.timestamp)
2528
+ link.transaction.setTimestamp(link.timestamp);
2529
+ }
2530
+ }
2531
+ // ImmediateTransaction: no clock pinning in lock(); save() stamps each write
2532
+ // from the committed handle floor for both scoped and hold handles.
2533
+ return this.#reloadLocked(id, handle);
2534
+ }, (err) => {
2535
+ link.unregisterPendingLock(primaryStore, keyId);
2536
+ throw err;
2537
+ });
2538
+ }
2539
+ #reloadLocked(id, holdHandle, preserveChanges = false) {
2540
+ // For freshness, read the committed entry (snapshot-free) so a hold lock sees concurrent
2541
+ // committed writes rather than a stale snapshot. A write earlier in THIS explicit
2542
+ // transaction has not landed in that committed entry yet (harper#1968: Harper defers an
2543
+ // explicit transaction's writes until the writing call actually runs them), so pull the
2544
+ // current value the same way a chained write picks up its basis (priorStagedWrite): the
2545
+ // record comes from the prior staged write, the rest of the entry (version, audit chain,
2546
+ // blob metadata) stays the pre-transaction one.
2547
+ const link = txnForContext(this.getContext());
2548
+ let entryForReload = primaryStore.getEntry(id);
2549
+ if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.OPEN) {
2550
+ const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
2551
+ const tailWrite = link.writesByKey?.get(primaryStore)?.get(keyId);
2552
+ const priorStaged = tailWrite && (tailWrite.stagedEntry !== undefined ? tailWrite : (0, DatabaseTransaction_ts_1.priorStagedWrite)(tailWrite));
2553
+ if (priorStaged?.stagedEntry !== undefined) {
2554
+ entryForReload = entryForReload
2555
+ ? { ...entryForReload, value: priorStaged.stagedEntry.value }
2556
+ : { value: priorStaged.stagedEntry.value };
2557
+ if (entryForReload.value && typeof entryForReload.value === 'object') {
2558
+ // Register the merged entry in entryMap so getUpdatedTime() works.
2559
+ RecordEncoder_ts_1.entryMap.set(entryForReload.value, entryForReload);
2560
+ }
2561
+ }
2562
+ }
2563
+ if ((0, DatabaseTransaction_ts_1.writeKeyId)(id) !== (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId())) {
2564
+ // lock(target) where target differs from this record: return a separate instance.
2565
+ const fresh = new this.constructor(id, this.getContext());
2566
+ _a._updateResource(fresh, entryForReload);
2567
+ if (holdHandle != null) {
2568
+ fresh.#lockHandle = holdHandle;
2569
+ // Do not clear this.#lockHandle: the original instance keeps its own lock on its
2570
+ // own id; the fresh instance owns the lock on the target id independently.
2571
+ }
2572
+ fresh.#lockWritable = true;
2573
+ // Scoped (not hold) stages exactly like update(): create the TransactionWrite now so
2574
+ // save() is the ordinary #savingOperation path. Hold keeps deferred staging (the
2575
+ // acquiring transaction may commit before the holder ever writes).
2576
+ if (!fresh.#lockHandle.hold)
2577
+ fresh._writeUpdate(id, fresh.#changes, false);
2578
+ return fresh;
2579
+ }
2580
+ // Store the handle for both scoped and hold locks; undefined (re-entrant hold fast-path)
2581
+ // must not clear a handle already set.
2582
+ if (holdHandle != null)
2583
+ this.#lockHandle = holdHandle;
2584
+ _a._updateResource(this, entryForReload);
2585
+ // Preserve staged changes when upgrading the same instance from scoped to hold so that
2586
+ // set() calls made under the scoped lock survive the reload.
2587
+ if (!preserveChanges)
2588
+ this.#changes = undefined;
2589
+ this.#lockWritable = true;
2590
+ // Scoped (not hold): stage now, same as update() would. Skip if a write from an earlier
2591
+ // lock() cycle on this instance is still pending (re-entrant call before its save()).
2592
+ if (!this.#lockHandle.hold && !this.#savingOperation)
2593
+ this._writeUpdate(id, this.#changes, false);
2594
+ return this;
2595
+ }
2596
+ /**
2597
+ * Release the lock this instance holds. Resolves true when this call cleared the native key lock.
2598
+ * Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
2599
+ * instance is no longer lock-writable; writes through it require a fresh lock.
2111
2600
  */
2112
- lock() {
2113
- throw new Error('Not yet implemented');
2601
+ unlock() {
2602
+ // Always clear the local lock-writable state so subsequent writes on this instance are
2603
+ // ungated, regardless of whether the handle was already released.
2604
+ const handle = this.#lockHandle;
2605
+ this.#lockHandle = undefined;
2606
+ this.#lockWritable = false;
2607
+ if (!handle || handle.released)
2608
+ return Promise.resolve(false);
2609
+ const link = txnForContext(this.getContext());
2610
+ // A scoped lock staged its write at lock() time; released before commit, that write must not
2611
+ // run into the released-handle guard at the sweep.
2612
+ if (this.#savingOperation && !this.#savingOperation.saved && this.#savingOperation.lockHandle === handle)
2613
+ this.#savingOperation = null;
2614
+ detachScopedUpgradeWrite(link, (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId()), handle);
2615
+ link.unregisterRecordLock(handle);
2616
+ return Promise.resolve(handle.release());
2114
2617
  }
2115
2618
  static operation(operation, context) {
2116
2619
  operation.table ||= tableName;
@@ -2226,8 +2729,10 @@ function makeTable(options) {
2226
2729
  // a notification that a write has already occurred in the canonical data source, we need to update our
2227
2730
  // local copy
2228
2731
  _writeUpdate(id, recordUpdate, fullUpdate, options) {
2732
+ this.#assertLiveHandle(id);
2229
2733
  const context = this.getContext();
2230
2734
  const transaction = txnForContext(context);
2735
+ const replaying = transaction.isReplay === true;
2231
2736
  checkValidId(id);
2232
2737
  if (fullUpdate && recordUpdate == null && options?.isNotification) {
2233
2738
  // A source/replication-applied put must carry the record; these applies skip record
@@ -2270,6 +2775,13 @@ function makeTable(options) {
2270
2775
  nodeName: context?.nodeName,
2271
2776
  fullUpdate,
2272
2777
  deferSave: true,
2778
+ // the origin's record version on an applied write; absent for a locally-originated one
2779
+ recordVersion: options?.version,
2780
+ // Include the lock handle (if any) so the expired-handle guard in
2781
+ // DatabaseTransaction.save() can throw 409 when the lease has lapsed.
2782
+ // Only attach the hold handle when it covers exactly this key; off-key writes
2783
+ // are ordinary and must not carry an unrelated hold's handle.
2784
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
2273
2785
  validate: (txnTime, committedBy = transaction) => {
2274
2786
  if (!recordUpdate)
2275
2787
  recordUpdate = this.#changes;
@@ -2394,6 +2906,7 @@ function makeTable(options) {
2394
2906
  this.#savingOperation = null;
2395
2907
  write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
2396
2908
  let omitLocalRecord = false;
2909
+ const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2397
2910
  // we use optimistic locking to only commit if the existing record state still holds true.
2398
2911
  // this is superior to using an async transaction since it doesn't require JS execution
2399
2912
  // during the write transaction.
@@ -2454,8 +2967,8 @@ function makeTable(options) {
2454
2967
  // best-effort keyed lookup in the capped block below — see #1148. precedesExistingVersion(...)
2455
2968
  // === 0 is the identity tie: same version AND same node (the local node is id 0, so an undefined
2456
2969
  // options?.nodeId resolves to the same 0 the ref stored).
2457
- if (existingEntry.additionalAuditRefs?.some((ref) => ref.version === txnTime &&
2458
- precedesExistingVersion(txnTime, { version: txnTime, localTime: txnTime, key: id, nodeId: ref.nodeId }, options?.nodeId) === 0)) {
2970
+ if (existingEntry.additionalAuditRefs?.some((ref) => ref.version === txnLogKey &&
2971
+ precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId }, options?.nodeId) === 0)) {
2459
2972
  write.skipped = true;
2460
2973
  return; // out-of-order write already folded into this record
2461
2974
  }
@@ -2483,10 +2996,10 @@ function makeTable(options) {
2483
2996
  if (!oldestRetainedAuditTimeResolved) {
2484
2997
  oldestRetainedAuditTimeResolved = true;
2485
2998
  // getRange yields ascending by audit-log key, so the first entry is the oldest retained.
2486
- // Mirror replicationConnection's retention check and the cleanup key basis (localTime ??
2487
- // version). Fall back to the nominal time-based purge floor when the log is empty/unavailable.
2999
+ // Mirror replicationConnection's retention check and the cleanup key basis (`txnLogKey`).
3000
+ // Fall back to the nominal time-based purge floor when the log is empty/unavailable.
2488
3001
  for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
2489
- oldestRetainedAuditTime = entry.localTime ?? entry.version;
3002
+ oldestRetainedAuditTime = entry.txnLogKey;
2490
3003
  break;
2491
3004
  }
2492
3005
  oldestRetainedAuditTime ??= Date.now() - auditStore_ts_1.auditRetention;
@@ -2499,43 +3012,67 @@ function makeTable(options) {
2499
3012
  // depth-cap block. This is the same keyed lookup that block performs, hoisted ahead of the walk.
2500
3013
  // It is what catches transitive/proxied re-deliveries: they arrive buried below the record head
2501
3014
  // (so replication's head-tie fast-skip can't see them) yet are exact duplicates. Keyed by nodeId,
2502
- // so it is correct across multiple source nodes. RocksDB-only: LMDB audit entries are keyed by
2503
- // local audit time, not version, so this version-keyed lookup doesn't apply there (LMDB keeps the
2504
- // exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
3015
+ // so it is correct across multiple source nodes. The lookup key is this write's LOG key, not its
3016
+ // record version — a replication apply commits under the origin's log key while storing the
3017
+ // origin's version, and only the log key addresses the entry (harper#2412).
3018
+ // RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
3019
+ // there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
2505
3020
  // simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
2506
3021
  // check above remains the read-your-writes guard. Never when this write staged in a prior
2507
3022
  // failed attempt: that attempt already appended this write's own audit entry, so the lookup
2508
3023
  // would find it and skip the write as "already applied" when the record was never committed.
2509
3024
  // A recommit of the same transaction survived that skip only because the old write batch
2510
3025
  // still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
2511
- if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnTime)) {
2512
- const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId);
3026
+ if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
3027
+ const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
2513
3028
  if (priorAudit &&
2514
- priorAudit.version === txnTime &&
2515
- precedesExistingVersion(txnTime, { version: txnTime, localTime: txnTime, key: id, nodeId: priorAudit.nodeId }, options?.nodeId) === 0) {
3029
+ priorAudit.txnLogKey === txnLogKey &&
3030
+ precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: priorAudit.nodeId }, options?.nodeId) === 0) {
2516
3031
  write.skipped = true;
2517
3032
  return; // duplicate already applied; avoid the resequencing walk
2518
3033
  }
2519
3034
  }
2520
3035
  // incremental CRDT updates are only available with audit logging on
2521
- let localTime = existingEntry.localTime;
3036
+ const initialAuditHead = isRocksDB
3037
+ ? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
3038
+ : { txnLogKey: existingEntry.localTime, nodeId: existingEntry.nodeId };
3039
+ let localTime = initialAuditHead.txnLogKey;
2522
3040
  let auditedVersion = existingEntry.version;
2523
3041
  logger_ts_1.logger.debug?.('Applying CRDT update to record with id: ', id, 'txn time', new Date(txnTime), 'applying later update from:', new Date(auditedVersion), 'local recorded time', new Date(localTime));
2524
- let nodeId = existingEntry.nodeId;
3042
+ let nodeId = initialAuditHead.nodeId;
2525
3043
  const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
2526
3044
  const auditRefsToVisit = existingEntry.additionalAuditRefs
2527
3045
  ? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
2528
3046
  : [];
2529
- // Collect any existing audit refs that should be preserved (those older than current transaction)
3047
+ // Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
2530
3048
  if (existingEntry.additionalAuditRefs) {
2531
3049
  for (const ref of existingEntry.additionalAuditRefs) {
2532
- if (ref.version <= txnTime) {
2533
- additionalAuditRefs.push(ref);
2534
- }
3050
+ additionalAuditRefs.push(ref);
2535
3051
  }
2536
3052
  }
2537
3053
  let addedAuditRef = false;
2538
3054
  let nextRef;
3055
+ const visitedAuditRefs = new Set();
3056
+ const queuePreviousAuditRefs = (auditRecord) => {
3057
+ const previousRefs = auditRecord.previousAdditionalAuditRefs;
3058
+ if (previousRefs) {
3059
+ for (const ref of previousRefs) {
3060
+ auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
3061
+ logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
3062
+ version: ref.version,
3063
+ nodeId: ref.nodeId,
3064
+ });
3065
+ }
3066
+ }
3067
+ };
3068
+ const advanceToPreviousAudit = (auditRecord) => {
3069
+ const previousRefs = auditRecord.previousAdditionalAuditRefs;
3070
+ const previousHead = isRocksDB && previousRefs?.length
3071
+ ? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, previousRefs)
3072
+ : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
3073
+ localTime = previousHead.txnLogKey;
3074
+ nodeId = previousHead.nodeId;
3075
+ };
2539
3076
  let walkSteps = 0;
2540
3077
  let auditWalkCapped = false;
2541
3078
  // Early-out residual: as we walk the chain newest-first, fold each succeeding patch into a
@@ -2554,17 +3091,21 @@ function makeTable(options) {
2554
3091
  // appended this write's own audit entry, so the lookup would match it while the record was
2555
3092
  // never committed (see the up-front keyed dedup above).
2556
3093
  const isReDeliveredDuplicate = () => {
2557
- if (stagedOwnAuditEntry)
3094
+ if (replaying || stagedOwnAuditEntry)
2558
3095
  return false;
2559
- if (!dedupVersionCouldBeRetained(txnTime))
2560
- return false; // pre-retention version — skip the end-of-log scan (best-effort; see above)
2561
- const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId);
3096
+ if (!dedupVersionCouldBeRetained(txnLogKey))
3097
+ return false; // pre-retention log key — skip the end-of-log scan (best-effort; see above)
3098
+ const duplicate = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
2562
3099
  return (duplicate &&
2563
- duplicate.version === txnTime &&
2564
- precedesExistingVersion(txnTime, { version: txnTime, localTime: txnTime, key: id, nodeId: duplicate.nodeId }, options?.nodeId) === 0);
3100
+ duplicate.txnLogKey === txnLogKey &&
3101
+ precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: duplicate.nodeId }, options?.nodeId) === 0);
2565
3102
  };
2566
3103
  do {
2567
3104
  while (localTime > txnTime || (auditedVersion >= txnTime && localTime > 0)) {
3105
+ const auditIdentity = `${nodeId ?? 0}:${localTime}`;
3106
+ if (visitedAuditRefs.has(auditIdentity))
3107
+ break;
3108
+ visitedAuditRefs.add(auditIdentity);
2568
3109
  // Bound the walk only for RocksDB, where the OOM was observed (issue #1114): each step
2569
3110
  // is a transaction-log range scan + msgpackr decode, and the per-node logs can be huge.
2570
3111
  // LMDB audit entries are keyed by local audit time (not version), so the duplicate
@@ -2576,19 +3117,39 @@ function makeTable(options) {
2576
3117
  const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
2577
3118
  if (!auditRecord)
2578
3119
  break;
3120
+ queuePreviousAuditRefs(auditRecord);
3121
+ if (isRocksDB &&
3122
+ !replaying &&
3123
+ !stagedOwnAuditEntry &&
3124
+ localTime === txnLogKey &&
3125
+ precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId }, options?.nodeId) === 0) {
3126
+ write.skipped = true;
3127
+ return;
3128
+ }
2579
3129
  auditedVersion = auditRecord.version;
2580
3130
  if (auditedVersion >= txnTime) {
2581
3131
  if (auditedVersion === txnTime) {
2582
3132
  precedesExisting = precedesExistingVersion(txnTime, { version: auditedVersion, localTime: localTime, key: id, nodeId: auditRecord.nodeId }, options?.nodeId);
2583
3133
  if (precedesExisting === 0) {
2584
- logger_ts_1.logger.debug?.('The transaction time is equal to the existing version, treating as duplicate', id);
2585
- write.skipped = true;
2586
- return; // treat a tie as a duplicate and drop it
3134
+ if (isRocksDB && localTime !== txnLogKey) {
3135
+ // Same origin and record version, but a distinct write. Its per-origin log key
3136
+ // orders the otherwise non-unique record clock without comparing keys across origins.
3137
+ precedesExisting = txnLogKey > localTime ? 1 : -1;
3138
+ }
3139
+ else if (replaying || stagedOwnAuditEntry) {
3140
+ // The log entry being replayed (or staged by this write's failed attempt) is
3141
+ // the write itself, not proof that its primary-store mutation committed.
3142
+ precedesExisting = 1;
3143
+ }
3144
+ else {
3145
+ logger_ts_1.logger.debug?.('The transaction time and log key match the existing write, treating as duplicate', id);
3146
+ write.skipped = true;
3147
+ return;
3148
+ }
2587
3149
  }
2588
3150
  if (precedesExisting > 0) {
2589
3151
  // if the existing version is older, we can skip this update
2590
- localTime = auditRecord.previousVersion;
2591
- nodeId = auditRecord.previousNodeId;
3152
+ advanceToPreviousAudit(auditRecord);
2592
3153
  continue;
2593
3154
  }
2594
3155
  }
@@ -2623,23 +3184,16 @@ function makeTable(options) {
2623
3184
  }
2624
3185
  if (!addedAuditRef && isRocksDB) {
2625
3186
  addedAuditRef = true;
2626
- // Add a reference to this older audit record if we had out-of-order writes
2627
- additionalAuditRefs.push({ version: txnTime, nodeId: options?.nodeId });
3187
+ // Add a reference to this older audit record if we had out-of-order writes. The stored
3188
+ // value is a LOG key, not a record version: every consumer follows it straight into
3189
+ // `auditStore.get` (see the `auditRefsToVisit` mapping above and below), and on an
3190
+ // applied write those two clocks differ.
3191
+ additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
2628
3192
  logger_ts_1.logger.debug?.('Adding additional audit ref for out-of-order write', {
2629
- version: txnTime,
3193
+ txnLogKey,
2630
3194
  nodeId: options?.nodeId,
2631
3195
  });
2632
3196
  }
2633
- // Collect any additional audit refs from this audit record to traverse other branches
2634
- if (auditRecord.previousAdditionalAuditRefs) {
2635
- for (const ref of auditRecord.previousAdditionalAuditRefs) {
2636
- auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
2637
- logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
2638
- version: ref.version,
2639
- nodeId: ref.nodeId,
2640
- });
2641
- }
2642
- }
2643
3197
  // Every field of this write is overwritten by newer writes, and there is no alternate
2644
3198
  // audit branch left to scan, so it is fully superseded — the same outcome as walking to
2645
3199
  // the end and taking the `writeCommit(false)` escape below, reached without paying the rest
@@ -2657,8 +3211,7 @@ function makeTable(options) {
2657
3211
  }
2658
3212
  return writeCommit(false);
2659
3213
  }
2660
- localTime = auditRecord.previousVersion;
2661
- nodeId = auditRecord.previousNodeId;
3214
+ advanceToPreviousAudit(auditRecord);
2662
3215
  }
2663
3216
  // Check if we need to scan additional audit refs from this record
2664
3217
  if (auditWalkCapped)
@@ -2753,14 +3306,14 @@ function makeTable(options) {
2753
3306
  throw new Error('Can not assign a record to a record, check for circular references');
2754
3307
  if (residencyId == undefined) {
2755
3308
  if (entry?.residencyId)
2756
- context.previousResidency = TableResource.getResidencyRecord(entry.residencyId);
2757
- const residency = residencyFromFunction(TableResource.getResidency(recordToStore, context));
3309
+ context.previousResidency = _a.getResidencyRecord(entry.residencyId);
3310
+ const residency = residencyFromFunction(_a.getResidency(recordToStore, context));
2758
3311
  if (residency) {
2759
3312
  if (!residency.includes(server.hostname)) {
2760
3313
  // if we aren't in the residency list, specify that our local record should be omitted or be partial
2761
3314
  auditRecordToStore ??= recordToStore;
2762
3315
  omitLocalRecord = true;
2763
- if (TableResource.getResidencyById) {
3316
+ if (_a.getResidencyById) {
2764
3317
  // complete omission of the record that doesn't belong here
2765
3318
  recordToStore = undefined;
2766
3319
  }
@@ -2830,7 +3383,17 @@ function makeTable(options) {
2830
3383
  }
2831
3384
  })());
2832
3385
  updateIndices(id, existingRecord, recordToStore, transaction && { transaction });
3386
+ // Preserve an addressable audit head when the record and log clocks diverge.
3387
+ if (isRocksDB && audit && !isCopyApply && txnLogKey !== txnTime) {
3388
+ const headIndex = additionalAuditRefs.findIndex((ref) => ref.version === txnLogKey && (ref.nodeId ?? 0) === (options?.nodeId ?? 0));
3389
+ if (headIndex > 0)
3390
+ additionalAuditRefs.unshift(additionalAuditRefs.splice(headIndex, 1)[0]);
3391
+ else if (headIndex < 0)
3392
+ additionalAuditRefs.unshift({ version: txnLogKey, nodeId: options?.nodeId });
3393
+ }
2833
3394
  writeCommit(true);
3395
+ if (write.trackRecordVersion)
3396
+ write.recordVersionApplied = true;
2834
3397
  if (expiresAt >= 0) {
2835
3398
  scheduleCleanup(); // arm for replicated writes too, not just local-context writes
2836
3399
  // A runtime per-record expiresAt on a table with no table-level expiration/eviction, no expiresAt
@@ -2857,6 +3420,8 @@ function makeTable(options) {
2857
3420
  user: context?.user,
2858
3421
  residencyId,
2859
3422
  expiresAt,
3423
+ recordVersion: txnTime,
3424
+ recordNodeId: precedesExisting < 0 ? existingEntry?.nodeId : options?.nodeId,
2860
3425
  nodeId: options?.nodeId,
2861
3426
  viaNodeId: options?.viaNodeId,
2862
3427
  originatingOperation: context?.originatingOperation,
@@ -2894,7 +3459,7 @@ function makeTable(options) {
2894
3459
  // calls the backend, and a tracked-instance mutation (update(id,{}); row.source=…;
2895
3460
  // save()) that sets the source via accessors after update() won't re-embed. A
2896
3461
  // resource-layer re-embed is the proper fix; tracked as a follow-up.
2897
- const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, TableResource.embedAttributes, TableResource.userEmbedders);
3462
+ const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, _a.embedAttributes, _a.userEmbedders);
2898
3463
  const proceed = () => {
2899
3464
  // On a source/replication apply (`isNotification`), the record's already-saved blobs were
2900
3465
  // received out-of-band for THIS write, so track them for skip/abort cleanup (harper-pro#406).
@@ -2950,6 +3515,7 @@ function makeTable(options) {
2950
3515
  return Boolean(this.#record);
2951
3516
  }
2952
3517
  _writeDelete(id, options) {
3518
+ this.#assertLiveHandle(id);
2953
3519
  const context = this.getContext();
2954
3520
  const transaction = txnForContext(context);
2955
3521
  checkValidId(id);
@@ -2960,6 +3526,8 @@ function makeTable(options) {
2960
3526
  entry,
2961
3527
  chainsStagedState: true,
2962
3528
  nodeName: context?.nodeName,
3529
+ recordVersion: options?.version,
3530
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
2963
3531
  before: this.constructor.source?.delete && !context?.source
2964
3532
  ? this.constructor.source.delete.bind(this.constructor.source, id, undefined, context)
2965
3533
  : undefined,
@@ -2971,10 +3539,11 @@ function makeTable(options) {
2971
3539
  const priorStagedOp = (0, DatabaseTransaction_ts_1.priorStagedWrite)(write);
2972
3540
  const priorStaged = priorStagedOp?.stagedEntry;
2973
3541
  const existingRecord = priorStaged ? priorStaged.value : existingEntry?.value;
3542
+ const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2974
3543
  if (retry) {
2975
3544
  if (context && existingEntry?.version > (context.lastModified || 0))
2976
3545
  context.lastModified = existingEntry.version;
2977
- TableResource._updateResource(this, existingEntry);
3546
+ _a._updateResource(this, existingEntry);
2978
3547
  }
2979
3548
  // a strictly newer record exists locally, so this delete loses. An earlier write in this
2980
3549
  // transaction can never trip this guard — it shares this transaction's timestamp and
@@ -2992,6 +3561,10 @@ function makeTable(options) {
2992
3561
  viaNodeId: options?.viaNodeId,
2993
3562
  transaction,
2994
3563
  tableToTrack: tableName,
3564
+ recordVersion: txnTime,
3565
+ additionalAuditRefs: isRocksDB && audit && txnLogKey !== txnTime
3566
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
3567
+ : undefined,
2995
3568
  }, 'delete');
2996
3569
  if (!audit || isRocksDB)
2997
3570
  scheduleCleanup();
@@ -3001,6 +3574,8 @@ function makeTable(options) {
3001
3574
  (0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
3002
3575
  }
3003
3576
  write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
3577
+ if (write.trackRecordVersion)
3578
+ write.recordVersionApplied = true;
3004
3579
  // the removal supersedes the nearest record an earlier write in this transaction stored
3005
3580
  // (older ones were already marked by their staged successors), so its saved blobs are
3006
3581
  // cleaned up post-commit unless its audit entry references them
@@ -3161,7 +3736,6 @@ function makeTable(options) {
3161
3736
  // objects. Entries are small and shallow; the clone is cheap next to the query.
3162
3737
  conditions = cloneConditions(conditions);
3163
3738
  let orderAlignedCondition;
3164
- let syntheticOrderCondition;
3165
3739
  const filtered = {};
3166
3740
  function prepareConditions(conditions, operator) {
3167
3741
  // some validation:
@@ -3260,7 +3834,7 @@ function makeTable(options) {
3260
3834
  // Note, that we do allow users to disable condition re-ordering, in case they have knowledge of a preferred
3261
3835
  // order for their query.
3262
3836
  if (conditions.length > 1 && operator !== 'or')
3263
- return sortBy(conditions, (0, search_ts_1.estimateCondition)(TableResource));
3837
+ return sortBy(conditions, (0, search_ts_1.estimateCondition)(_a));
3264
3838
  else
3265
3839
  return conditions;
3266
3840
  }
@@ -3301,7 +3875,7 @@ function makeTable(options) {
3301
3875
  // if it is indexed, we add a pseudo-condition to align with the natural sort order of the index.
3302
3876
  // the primary key has no secondary index, but the primary store is itself keyed in
3303
3877
  // primary-key order, so scanning it is already aligned with the sort
3304
- orderAlignedCondition = syntheticOrderCondition = { ...sort, comparator: 'sort' };
3878
+ orderAlignedCondition = { ...sort, comparator: 'sort' };
3305
3879
  conditions.push(orderAlignedCondition);
3306
3880
  }
3307
3881
  else if (conditions.length === 0 && !target.allowFullScan)
@@ -3328,15 +3902,17 @@ function makeTable(options) {
3328
3902
  }
3329
3903
  }
3330
3904
  else {
3331
- // if we had to add an aligned condition that isn't first, we remove it and do ordering later —
3332
- // only the one we added; a caller's own condition on the sort attribute is still a filter
3333
- const syntheticIndex = syntheticOrderCondition ? conditions.indexOf(syntheticOrderCondition) : -1;
3334
- if (syntheticIndex >= 0)
3335
- conditions.splice(syntheticIndex, 1);
3905
+ // if we had to add an aligned condition that isn't first, we remove it and do ordering later
3906
+ if (orderAlignedCondition)
3907
+ conditions.splice(conditions.indexOf(orderAlignedCondition), 1);
3336
3908
  postOrdering = sort;
3337
3909
  }
3338
3910
  }
3339
3911
  const select = target.select;
3912
+ // Whether the caller supplied real filter conditions — read from the raw request, NOT the
3913
+ // planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
3914
+ // injected full-scan condition). Used to pick the count-estimate source below.
3915
+ const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
3340
3916
  if (conditions.length === 0) {
3341
3917
  conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
3342
3918
  }
@@ -3373,7 +3949,7 @@ function makeTable(options) {
3373
3949
  const recordAccess = boundRowFilter || typeof target.vectorFilter === 'function'
3374
3950
  ? { rowFilter: boundRowFilter, vectorFilter: target.vectorFilter }
3375
3951
  : undefined;
3376
- const entries = (0, search_ts_1.executeConditions)(conditions, operator, TableResource, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
3952
+ const entries = (0, search_ts_1.executeConditions)(conditions, operator, _a, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
3377
3953
  const ensure_loaded = target.ensureLoaded !== false;
3378
3954
  // The guards inside executeConditions evaluate the
3379
3955
  // LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
@@ -3390,11 +3966,126 @@ function makeTable(options) {
3390
3966
  // since those never run the ensureLoaded-gated freshness check this transform
3391
3967
  // otherwise applies unconditionally to every read.
3392
3968
  const includeExpired = target.includeExpired === true;
3393
- const transformToRecord = TableResource.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, boundRowFilter, includeExpired, postOrdering);
3394
- let results = TableResource.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
3969
+ const transformToRecord = _a.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, boundRowFilter, includeExpired, postOrdering);
3970
+ let results = _a.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
3971
+ const offset = target.offset || 0;
3972
+ const end = target.limit !== undefined ? offset + target.limit : undefined;
3973
+ // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
3974
+ // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
3975
+ // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
3976
+ // table estimate. Opt-in only — the default streaming path below is untouched.
3977
+ //
3978
+ // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
3979
+ // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
3980
+ // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
3981
+ // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
3982
+ // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
3983
+ // normal streaming path with no count. This bounds the offset too: without it a huge offset would
3984
+ // postpone the exact guardrail (which only engages past the page) until that offset was scanned.
3985
+ const pageLimit = target.limit;
3986
+ if (target.count &&
3987
+ Number.isInteger(pageLimit) &&
3988
+ pageLimit >= 0 &&
3989
+ pageLimit <= MAX_COUNT_PAGE &&
3990
+ Number.isInteger(offset) &&
3991
+ offset >= 0 &&
3992
+ offset + pageLimit <= MAX_EXACT_COUNT_SCAN) {
3993
+ const wantExact = target.count === 'exact';
3994
+ const pageEnd = offset + pageLimit;
3995
+ const countStart = node_perf_hooks_1.performance.now();
3996
+ // A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
3997
+ // chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
3998
+ // the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
3999
+ // different `count=exact` totals. Any query whose execution touches a custom index is affected: a
4000
+ // custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
4001
+ // filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
4002
+ // filter. Report the total as unavailable for those rather than advertising it as count=exact
4003
+ // (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
4004
+ // sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
4005
+ const touchesCustomIndex = (conds) => conds.some((c) => {
4006
+ if (!c)
4007
+ return false;
4008
+ if (c.conditions)
4009
+ return touchesCustomIndex(c.conditions);
4010
+ const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
4011
+ return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
4012
+ });
4013
+ const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
4014
+ return (async () => {
4015
+ const page = [];
4016
+ let scanned = 0;
4017
+ let exact = true;
4018
+ try {
4019
+ for await (const record of results) {
4020
+ if (scanned >= offset && scanned < pageEnd)
4021
+ page.push(record);
4022
+ scanned++;
4023
+ // A store whose async iterator settles synchronously (the common indexed-scan case) would
4024
+ // otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
4025
+ // for the whole count. Yield to the macrotask queue periodically so concurrent requests and
4026
+ // I/O still make progress during a large exact scan.
4027
+ if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0)
4028
+ await new Promise((resolve) => setImmediate(resolve));
4029
+ // The page window [offset, pageEnd) is always collected in full first — the guardrail
4030
+ // only ever abandons the running TOTAL, never truncates the page body.
4031
+ if (scanned >= pageEnd) {
4032
+ // `estimated` needs nothing past the page; an approximate (vector) exact total is going to
4033
+ // be reported unavailable anyway, so don't drain its tail for a number we won't publish.
4034
+ if (!wantExact || approximateResultSet)
4035
+ break;
4036
+ // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
4037
+ // large match set can't turn a bounded page fetch into an unbounded scan.
4038
+ if (scanned > MAX_EXACT_COUNT_SCAN || node_perf_hooks_1.performance.now() - countStart > MAX_EXACT_COUNT_MS) {
4039
+ exact = false;
4040
+ break;
4041
+ }
4042
+ }
4043
+ }
4044
+ }
4045
+ finally {
4046
+ // We own the iteration here (no results.onDone consumer), so release the read
4047
+ // transaction unconditionally — including when the drain throws — or the snapshot leaks.
4048
+ txn.doneReadTxn();
4049
+ }
4050
+ let total;
4051
+ if (wantExact) {
4052
+ // `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
4053
+ // an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
4054
+ total = exact && !approximateResultSet ? scanned : null;
4055
+ }
4056
+ else if (boundRowFilter || typeof target.vectorFilter === 'function') {
4057
+ // An opaque row/vector filter shapes the result but isn't reflected in the index/condition
4058
+ // estimate; guessing would both mislead and disclose cardinality the filter hides.
4059
+ total = null;
4060
+ }
4061
+ else if (!hasUserConditions) {
4062
+ total = (0, search_ts_1.estimatedEntryCount)(primaryStore);
4063
+ }
4064
+ else {
4065
+ // Estimate from the real conditions only — drop the planner's synthetic `sort`
4066
+ // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
4067
+ const est = (0, search_ts_1.estimateCondition)(_a)({
4068
+ conditions: conditions.filter((c) => c.comparator !== 'sort'),
4069
+ operator: operator ? String(operator).toLowerCase() : 'and',
4070
+ });
4071
+ total = isFinite(est) ? Math.round(est) : null;
4072
+ }
4073
+ // For an estimate, never report a total below the last row actually returned — keeps the
4074
+ // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
4075
+ // Exact totals are authoritative (and an empty page past the end must not be clamped up).
4076
+ if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
4077
+ total = offset + page.length;
4078
+ }
4079
+ page.recordCount = total;
4080
+ page.recordCountExact = wantExact && exact && !approximateResultSet;
4081
+ page.selectApplied = true;
4082
+ page.getColumns = getColumns;
4083
+ return page;
4084
+ })();
4085
+ }
3395
4086
  // apply any offset/limit after all the sorting and filtering
3396
4087
  if (target.offset || target.limit !== undefined)
3397
- results = results.slice(target.offset, target.limit !== undefined ? (target.offset || 0) + target.limit : undefined);
4088
+ results = results.slice(offset, end);
3398
4089
  results.onDone = () => {
3399
4090
  results.onDone = null; // ensure that it isn't called twice
3400
4091
  txn.doneReadTxn();
@@ -3709,12 +4400,12 @@ function makeTable(options) {
3709
4400
  if (resolver.directReturn)
3710
4401
  return callback(value, attribute_name);
3711
4402
  if (value && typeof value === 'object') {
3712
- const targetTable = resolver.definition?.tableClass || TableResource;
4403
+ const targetTable = resolver.definition?.tableClass || _a;
3713
4404
  if (!transformCache)
3714
4405
  transformCache = {};
3715
4406
  // Use the target table's own read transaction; each table's readTxn is
3716
4407
  // scoped to its RocksDB column family and cannot read another table's store.
3717
- const targetReadTxn = targetTable === TableResource ? readTxn : targetTable._readTxnForContext(context);
4408
+ const targetReadTxn = targetTable === _a ? readTxn : targetTable._readTxnForContext(context);
3718
4409
  const transform = transformCache[attribute_name] ||
3719
4410
  (transformCache[attribute_name] = targetTable.transformEntryForSelect(
3720
4411
  // if it is a simple string, there is no select for the next level,
@@ -3767,7 +4458,7 @@ function makeTable(options) {
3767
4458
  else {
3768
4459
  value = record[attribute_name];
3769
4460
  if (value && typeof value === 'object' && attribute_name !== attribute) {
3770
- const subTransform = TableResource.transformEntryForSelect(attribute.select || attribute, context, readTxn, null);
4461
+ const subTransform = _a.transformEntryForSelect(attribute.select || attribute, context, readTxn, null);
3771
4462
  // Plain JSON nested values: arrays project per-element so that
3772
4463
  // `select: [{ name: 'addresses', select: ['city'] }]` returns
3773
4464
  // `addresses: [{ city }, { city }]` rather than a single object.
@@ -3846,6 +4537,10 @@ function makeTable(options) {
3846
4537
  if (!auditStore)
3847
4538
  throw new Error('Can not subscribe to a table without an audit log');
3848
4539
  if (!audit) {
4540
+ // Turning auditing on is a schema write, and a branch's Table classes carry the base's
4541
+ // logical name: without this a subscribe through a branched application would enable
4542
+ // auditing on the live base table for every other consumer, with no DDL call involved.
4543
+ _a.assertSchemaMutable('enable auditing for a subscription');
3849
4544
  (0, databases_ts_1.table)({ table: tableName, database: databaseName, schemaDefined, attributes, audit: true });
3850
4545
  }
3851
4546
  const getFullRecord = !request.rawEvents;
@@ -3901,7 +4596,7 @@ function makeTable(options) {
3901
4596
  return evaluateFilter(rowFilter, event.value, 'rowFilter');
3902
4597
  }
3903
4598
  : null;
3904
- const subscription = (0, transactionBroadcast_ts_1.addSubscription)(TableResource, thisId, function (id, auditRecord, localTime, beginTxn) {
4599
+ const subscription = (0, transactionBroadcast_ts_1.addSubscription)(_a, thisId, function (id, auditRecord, txnLogKey, beginTxn) {
3905
4600
  if (dropDuringReplay)
3906
4601
  return;
3907
4602
  try {
@@ -3928,8 +4623,7 @@ function makeTable(options) {
3928
4623
  // been written, so are fresh in memory.
3929
4624
  const entry = primaryStore.getEntry(id);
3930
4625
  if (entry) {
3931
- // staleness is a record-version comparison; auditRecord.version is the log key on RocksDB
3932
- if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version))
4626
+ if (entry.version !== auditRecord.version)
3933
4627
  return; // out of order event, with old update, don't send anything
3934
4628
  value = entry.value;
3935
4629
  type = entry.metadataFlags & exports.INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
@@ -3940,7 +4634,7 @@ function makeTable(options) {
3940
4634
  }
3941
4635
  const event = {
3942
4636
  id,
3943
- localTime,
4637
+ localTime: txnLogKey,
3944
4638
  value,
3945
4639
  version: auditRecord.version,
3946
4640
  type,
@@ -4009,10 +4703,10 @@ function makeTable(options) {
4009
4703
  continue;
4010
4704
  const id = auditRecord.recordId;
4011
4705
  if (thisId == null || isDescendantId(thisId, id)) {
4012
- const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.localTime);
4706
+ const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
4013
4707
  if (!send({
4014
4708
  id,
4015
- localTime: auditRecord.localTime,
4709
+ localTime: auditRecord.txnLogKey,
4016
4710
  value,
4017
4711
  version: auditRecord.version,
4018
4712
  type: auditRecord.type,
@@ -4025,7 +4719,7 @@ function makeTable(options) {
4025
4719
  return;
4026
4720
  }
4027
4721
  }
4028
- subscription.startTime = auditRecord.localTime ?? auditRecord.version; // update so we don't double send
4722
+ subscription.startTime = auditRecord.txnLogKey; // update so we don't double send
4029
4723
  }
4030
4724
  }
4031
4725
  finally {
@@ -4059,10 +4753,10 @@ function makeTable(options) {
4059
4753
  logger_ts_1.logger.warn?.(`previousCount backfill on ${tableName} stopped after inspecting ${MAX_PREVIOUS_COUNT_SCAN} in-scope audit records without collecting ${request.previousCount} accepted event(s); returning ${history.length} instead`);
4060
4754
  break;
4061
4755
  }
4062
- const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.localTime);
4756
+ const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
4063
4757
  const historyEntry = {
4064
4758
  id,
4065
- localTime: auditRecord.localTime,
4759
+ localTime: auditRecord.txnLogKey,
4066
4760
  value,
4067
4761
  version: auditRecord.version,
4068
4762
  type: auditRecord.type,
@@ -4079,7 +4773,7 @@ function makeTable(options) {
4079
4773
  }
4080
4774
  }
4081
4775
  catch (error) {
4082
- logger_ts_1.logger.error?.('Error getting history entry', auditRecord.localTime, error);
4776
+ logger_ts_1.logger.error?.('Error getting history entry', auditRecord.txnLogKey, error);
4083
4777
  }
4084
4778
  }
4085
4779
  for (let i = history.length; i > 0;) {
@@ -4167,6 +4861,12 @@ function makeTable(options) {
4167
4861
  logger_ts_1.logger.trace?.('re-retrieved record', localTime, this.#entry?.localTime);
4168
4862
  localTime = entry?.localTime;
4169
4863
  }
4864
+ let nodeId = entry?.nodeId;
4865
+ if (isRocksDB && entry) {
4866
+ const head = resolveAuditHead(thisId, entry.version, nodeId, entry.additionalAuditRefs);
4867
+ localTime = head.txnLogKey;
4868
+ nodeId = head.nodeId;
4869
+ }
4170
4870
  logger_ts_1.logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
4171
4871
  if (startTime < localTime) {
4172
4872
  // start time specified, get the audit history for this record. Set startTime up
@@ -4177,7 +4877,6 @@ function makeTable(options) {
4177
4877
  const history = [];
4178
4878
  let inspected = 0;
4179
4879
  let nextTime = localTime;
4180
- let nodeId = entry?.nodeId;
4181
4880
  do {
4182
4881
  if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
4183
4882
  recordsSinceYield = 0;
@@ -4208,8 +4907,11 @@ function makeTable(options) {
4208
4907
  else if (!isActive())
4209
4908
  return;
4210
4909
  }
4211
- nextTime = auditRecord.previousVersion;
4212
- nodeId = auditRecord.previousNodeId;
4910
+ const previousHead = isRocksDB
4911
+ ? resolveAuditHead(thisId, auditRecord.previousVersion, auditRecord.previousNodeId, auditRecord.previousAdditionalAuditRefs)
4912
+ : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
4913
+ nextTime = previousHead.txnLogKey;
4914
+ nodeId = previousHead.nodeId;
4213
4915
  }
4214
4916
  else
4215
4917
  break;
@@ -4404,6 +5106,7 @@ function makeTable(options) {
4404
5106
  store: primaryStore,
4405
5107
  entry: this.#entry,
4406
5108
  nodeName: context?.nodeName,
5109
+ recordVersion: options?.version,
4407
5110
  validate: () => {
4408
5111
  if (!context?.source) {
4409
5112
  transaction.checkOverloaded();
@@ -4617,6 +5320,7 @@ function makeTable(options) {
4617
5320
  return this.#version;
4618
5321
  }
4619
5322
  static async addAttributes(attributesToAdd) {
5323
+ _a.assertSchemaMutable('add attributes');
4620
5324
  const new_attributes = attributes.slice(0);
4621
5325
  for (const attribute of attributesToAdd) {
4622
5326
  if (!attribute.name)
@@ -4632,9 +5336,10 @@ function makeTable(options) {
4632
5336
  schemaDefined,
4633
5337
  attributes: new_attributes,
4634
5338
  });
4635
- return TableResource.indexingOperation;
5339
+ return _a.indexingOperation;
4636
5340
  }
4637
5341
  static async removeAttributes(names) {
5342
+ _a.assertSchemaMutable('remove attributes');
4638
5343
  const new_attributes = attributes.filter((attribute) => !names.includes(attribute.name));
4639
5344
  (0, databases_ts_1.table)({
4640
5345
  table: tableName,
@@ -4642,7 +5347,7 @@ function makeTable(options) {
4642
5347
  schemaDefined,
4643
5348
  attributes: new_attributes,
4644
5349
  });
4645
- return TableResource.indexingOperation;
5350
+ return _a.indexingOperation;
4646
5351
  }
4647
5352
  /**
4648
5353
  * Get the size of the table in bytes (based on amount of pages stored in the database)
@@ -4655,6 +5360,10 @@ function makeTable(options) {
4655
5360
  const stats = primaryStore.getStats();
4656
5361
  return (stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize;
4657
5362
  }
5363
+ /** Sizes of this table's durable record-structure dictionaries. */
5364
+ static getStructureCounts() {
5365
+ return primaryStore.encoder?.getStructureCounts?.();
5366
+ }
4658
5367
  static getAuditSize() {
4659
5368
  const stats = auditStore?.getStats();
4660
5369
  return (stats &&
@@ -4673,7 +5382,7 @@ function makeTable(options) {
4673
5382
  // iterate through the metadata entries to exclude their count and exclude the deletion counts
4674
5383
  const exactCount = options?.exactCount;
4675
5384
  const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
4676
- const start = performance.now();
5385
+ const start = node_perf_hooks_1.performance.now();
4677
5386
  // `entryCount` (the exact key count) is only needed once the scan blows the time budget --
4678
5387
  // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full
4679
5388
  // key-only scan, so we defer it: tables that finish within budget (the common case) and
@@ -4690,7 +5399,7 @@ function makeTable(options) {
4690
5399
  recordCount++;
4691
5400
  entriesScanned++;
4692
5401
  await rest();
4693
- if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) {
5402
+ if (!exactCount && !completeForExact && node_perf_hooks_1.performance.now() - start > TIME_LIMIT) {
4694
5403
  if (!counted) {
4695
5404
  counted = true;
4696
5405
  entryCount = isRocksDB
@@ -4814,7 +5523,7 @@ function makeTable(options) {
4814
5523
  const computed = attribute.computed;
4815
5524
  // Register the default embedder unless an author override is set. Sits outside
4816
5525
  // the resolver chain below so `@embed` fields still flow through auto-HNSW indexing.
4817
- if (attribute.embed && !TableResource.userSetEmbedders.has(attribute.name)) {
5526
+ if (attribute.embed && !_a.userSetEmbedders.has(attribute.name)) {
4818
5527
  this.userEmbedders[attribute.name] = (0, embedHook_ts_1.createDefaultEmbedder)(attribute.embed);
4819
5528
  }
4820
5529
  if (relationship) {
@@ -4880,7 +5589,7 @@ function makeTable(options) {
4880
5589
  const options = { transaction: txnForContext(context).getReadTxn() };
4881
5590
  const results = normalizedIds.map((id) => {
4882
5591
  const value = store[method](id, options);
4883
- if (TableResource.loadAsInstance === false)
5592
+ if (_a.loadAsInstance === false)
4884
5593
  freezeRecord(returnEntry ? value?.value : value);
4885
5594
  return value;
4886
5595
  });
@@ -4889,7 +5598,7 @@ function makeTable(options) {
4889
5598
  const value = definition.tableClass.primaryStore[returnEntry ? 'getEntry' : 'getSync'](ids, {
4890
5599
  transaction: txnForContext(context).getReadTxn(),
4891
5600
  });
4892
- if (TableResource.loadAsInstance === false)
5601
+ if (_a.loadAsInstance === false)
4893
5602
  freezeRecord(returnEntry ? value?.value : value);
4894
5603
  return value;
4895
5604
  };
@@ -5044,30 +5753,85 @@ function makeTable(options) {
5044
5753
  this.userSetEmbedders.add(attribute_name);
5045
5754
  }
5046
5755
  static async deleteHistory(endTime = 0, cleanupDeletedRecords = false) {
5047
- let completion;
5756
+ const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
5757
+ const inFlightRemovals = new Set();
5758
+ const removalSlotWaiters = [];
5759
+ let removalsAttempted = 0;
5760
+ let removalsSucceeded = 0;
5761
+ let firstRemovalError;
5762
+ function startRemoval(remove, errorMessage, onSuccess) {
5763
+ removalsAttempted++;
5764
+ const removal = new Promise((resolve) => resolve(remove()))
5765
+ .then(() => {
5766
+ removalsSucceeded++;
5767
+ onSuccess?.();
5768
+ }, (error) => {
5769
+ // capture before logging: a throwing logger must not cost us the error we may rethrow
5770
+ if (firstRemovalError === undefined)
5771
+ firstRemovalError = error;
5772
+ harper_logger_ts_1.default.warn(errorMessage, error);
5773
+ })
5774
+ .catch(() => undefined)
5775
+ .finally(() => {
5776
+ inFlightRemovals.delete(removal);
5777
+ removalSlotWaiters.shift()?.();
5778
+ });
5779
+ inFlightRemovals.add(removal);
5780
+ }
5781
+ function queueRemoval(remove, errorMessage, onSuccess) {
5782
+ if (inFlightRemovals.size >= maxConcurrentRemovals) {
5783
+ return new Promise((resolve) => {
5784
+ removalSlotWaiters.push(resolve);
5785
+ }).then(() => startRemoval(remove, errorMessage, onSuccess));
5786
+ }
5787
+ startRemoval(remove, errorMessage, onSuccess);
5788
+ }
5789
+ const drainRemovals = () => Promise.all(inFlightRemovals);
5048
5790
  let entriesDeleted = 0;
5049
- for (const auditRecord of auditStore.getRange({
5050
- start: 0,
5051
- end: endTime,
5052
- })) {
5053
- await rest(); // yield to other async operations
5054
- if (auditRecord.tableId !== tableId)
5055
- continue;
5056
- completion = (0, auditStore_ts_1.removeAuditEntry)(auditStore, auditRecord);
5057
- entriesDeleted++;
5791
+ try {
5792
+ for (const auditRecord of auditStore.getRange({
5793
+ start: 1, // must not be zero; see getHistory below for why
5794
+ end: endTime,
5795
+ })) {
5796
+ await rest(); // yield to other async operations
5797
+ if (auditRecord.tableId !== tableId)
5798
+ continue;
5799
+ const backpressure = queueRemoval(() => (0, auditStore_ts_1.removeAuditEntry)(auditStore, auditRecord), 'Error removing audit entry during deleteHistory', () => {
5800
+ entriesDeleted++;
5801
+ });
5802
+ if (backpressure)
5803
+ await backpressure;
5804
+ }
5805
+ }
5806
+ finally {
5807
+ await drainRemovals();
5058
5808
  }
5059
5809
  if (cleanupDeletedRecords) {
5060
5810
  // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't
5061
5811
  // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records
5062
- for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
5063
- const { value, localTime } = entry;
5064
- await rest(); // yield to other async operations
5065
- if (value === null && localTime < endTime) {
5066
- completion = (0, RecordEncoder_ts_1.removeEntry)(primaryStore, entry);
5812
+ try {
5813
+ for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
5814
+ const { key, value, localTime, version } = entry;
5815
+ await rest(); // yield to other async operations
5816
+ const auditTime = isRocksDB && version != null
5817
+ ? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
5818
+ : localTime;
5819
+ if (value === null && version != null && auditTime < endTime) {
5820
+ const backpressure = queueRemoval(() => primaryStore.remove(key, version), 'Error removing deleted record during deleteHistory');
5821
+ if (backpressure)
5822
+ await backpressure;
5823
+ }
5067
5824
  }
5068
5825
  }
5826
+ finally {
5827
+ await drainRemovals();
5828
+ }
5829
+ }
5830
+ if (removalsAttempted > 0 && removalsSucceeded === 0) {
5831
+ // zero progress must not report the same success as "nothing was eligible" (see DESIGN.md);
5832
+ // partial failures stay best-effort, logged and excluded from the returned count
5833
+ throw firstRemovalError ?? new Error('Every removal attempted during deleteHistory failed');
5069
5834
  }
5070
- await completion;
5071
5835
  return entriesDeleted;
5072
5836
  }
5073
5837
  static async *getHistory(startTime = 0, endTime = Infinity) {
@@ -5080,10 +5844,11 @@ function makeTable(options) {
5080
5844
  continue;
5081
5845
  yield {
5082
5846
  id: auditRecord.recordId,
5083
- localTime: auditRecord.version,
5847
+ // Compatibility-facing LMDB history has always reported/grouped by record version.
5848
+ localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
5084
5849
  version: auditRecord.version,
5085
5850
  type: auditRecord.type,
5086
- value: auditRecord.getValue(primaryStore, true, auditRecord.version),
5851
+ value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
5087
5852
  user: auditRecord.user,
5088
5853
  operation: auditRecord.originatingOperation,
5089
5854
  };
@@ -5096,7 +5861,9 @@ function makeTable(options) {
5096
5861
  const entry = primaryStore.getEntry(id);
5097
5862
  if (!entry)
5098
5863
  return history;
5099
- let nextVersion = entry.localTime;
5864
+ let nextVersion = isRocksDB
5865
+ ? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
5866
+ : entry.localTime;
5100
5867
  if (!nextVersion)
5101
5868
  throw new Error('The entry does not have a local audit time');
5102
5869
  const count = 0;
@@ -5110,17 +5877,20 @@ function makeTable(options) {
5110
5877
  if (auditRecord.tableId === tableId && (0, ordered_binary_1.compareKeys)(auditRecord.recordId, id) === 0) {
5111
5878
  history.splice(insertionPoint, 0, {
5112
5879
  id: auditRecord.recordId,
5113
- localTime: auditRecord.version,
5880
+ localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
5114
5881
  version: auditRecord.version,
5115
5882
  type: auditRecord.type,
5116
- // reconstruct each entry's record image as of its own version, not the audit
5883
+ // reconstruct each entry's record image as of its own log position, not the audit
5117
5884
  // window boundary (nextVersion), matching getHistory (issue #1330)
5118
- value: auditRecord.getValue(primaryStore, true, auditRecord.version),
5885
+ value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
5119
5886
  user: auditRecord.user,
5120
5887
  operation: auditRecord.originatingOperation,
5121
5888
  });
5122
- if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) {
5123
- highestPreviousVersion = auditRecord.previousVersion;
5889
+ const previousVersion = isRocksDB
5890
+ ? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, auditRecord.previousAdditionalAuditRefs).txnLogKey
5891
+ : auditRecord.previousVersion;
5892
+ if (previousVersion > highestPreviousVersion && previousVersion < start) {
5893
+ highestPreviousVersion = previousVersion;
5124
5894
  }
5125
5895
  }
5126
5896
  }
@@ -5139,13 +5909,20 @@ function makeTable(options) {
5139
5909
  }
5140
5910
  return Promise.all(promises);
5141
5911
  }
5912
+ /** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
5142
5913
  static cleanup() {
5914
+ disposed = true;
5915
+ clearTimeout(cleanupTimer);
5916
+ settlePendingCleanup();
5917
+ clearInterval(recordExpirationInterval);
5143
5918
  deleteCallbackHandle?.remove();
5919
+ (0, storageReclamation_ts_1.removeStorageReclamationHandler)(primaryStore.path, reclamationHandler);
5144
5920
  }
5145
5921
  static _readTxnForContext(context) {
5146
5922
  return txnForContext(context).getReadTxn();
5147
5923
  }
5148
5924
  }
5925
+ _a = TableResource;
5149
5926
  const throttledCallToSource = (0, throttle_ts_1.throttle)(async (source, id, sourceContext, existingEntry) => {
5150
5927
  // call the data source if it exists and will fulfill our request for data
5151
5928
  if (source && source.get && (!source.get.reliesOnPrototype || source.prototype.get)) {
@@ -5159,11 +5936,17 @@ function makeTable(options) {
5159
5936
  }, () => {
5160
5937
  throw new hdbError_ts_1.ServerError('Service unavailable, exceeded request queue limit for resolving cache record', 503);
5161
5938
  });
5162
- TableResource.updatedAttributes(); // on creation, update accessors as well
5163
- if (expirationMs)
5164
- TableResource.setTTLExpiration(expirationMs / 1000);
5165
- if (expiresAtProperty)
5166
- runRecordExpirationEviction();
5939
+ try {
5940
+ TableResource.updatedAttributes(); // on creation, update accessors as well
5941
+ if (expirationMs)
5942
+ TableResource.setTTLExpiration(expirationMs / 1000);
5943
+ if (expiresAtProperty)
5944
+ runRecordExpirationEviction();
5945
+ }
5946
+ catch (error) {
5947
+ TableResource.cleanup();
5948
+ throw error;
5949
+ }
5167
5950
  return TableResource;
5168
5951
  function updateIndices(id, existingRecord, record, options) {
5169
5952
  let hasChanges;
@@ -5590,6 +6373,22 @@ function makeTable(options) {
5590
6373
  return transaction;
5591
6374
  }
5592
6375
  }
6376
+ /**
6377
+ * Detach an unsaved TransactionWrite that a scoped lock() eagerly staged (see #reloadLocked)
6378
+ * once its handle upgrades to hold: hold staging is deferred and explicit-save-only, so a
6379
+ * dangling scoped write would otherwise auto-commit at the transaction sweep and clobber
6380
+ * whatever the hold write lands. Marking it .dropped lets a later save() on the instance that
6381
+ * owns it (checked via #savingOperation === this write) fall through to the hold branch
6382
+ * instead of resolving a detached, dead reference.
6383
+ */
6384
+ function detachScopedUpgradeWrite(link, keyId, handle) {
6385
+ for (const write of link.writes) {
6386
+ if (write && !write.saved && write.lockHandle === handle && (0, DatabaseTransaction_ts_1.writeKeyId)(write.key) === keyId) {
6387
+ write.dropped = true;
6388
+ link.detachWrite(write);
6389
+ }
6390
+ }
6391
+ }
5593
6392
  function getAttributeValue(entry, attribute_name, context, sort) {
5594
6393
  if (!entry) {
5595
6394
  return;
@@ -5707,6 +6506,10 @@ function makeTable(options) {
5707
6506
  async function getFromSource(source, id, existingEntry, context, target) {
5708
6507
  const metadataFlags = existingEntry?.metadataFlags;
5709
6508
  const existingVersion = existingEntry?.version;
6509
+ const existingRecord = existingEntry?.value;
6510
+ const inheritedTimestamp = context?.timestamp || context?.transaction?.timestamp;
6511
+ const sourceTimestamp = inheritedTimestamp ||
6512
+ (isRocksDB ? primaryStore.getMonotonicTimestamp() : (0, commonUtility_ts_1.getNextMonotonicTime)());
5710
6513
  let whenResolved, timer;
5711
6514
  // We start by locking the record so that there is only one resolution happening at once;
5712
6515
  // if there is already a resolution in process, we want to use the results of that resolution
@@ -5741,10 +6544,8 @@ function makeTable(options) {
5741
6544
  }
5742
6545
  // lock acquired — this request will actually load from source
5743
6546
  setLoadedFromSource(target, true);
5744
- const existingRecord = existingEntry?.value;
5745
6547
  // it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
5746
- // with the canonical data, we are simply fulfilling our local copy of the canonical data, but still don't
5747
- // want a timestamp later than the current transaction
6548
+ // with the canonical data, we are simply fulfilling our local copy of the canonical data.
5748
6549
  // we create a new context for the source, we want to determine the timestamp and don't want to
5749
6550
  // attribute this to the current user
5750
6551
  const sourceContext = {
@@ -5779,15 +6580,36 @@ function makeTable(options) {
5779
6580
  // belt to that suspenders, at the cost of a bounded wait on a merely slow source
5780
6581
  // before the drain's fail-closed timeout below.
5781
6582
  const commitPromise = (0, transaction_ts_1.transaction)(sourceContext, async (_txn) => {
5782
- const start = performance.now();
5783
- let updatedRecord;
6583
+ const start = node_perf_hooks_1.performance.now();
6584
+ let updatedRecord, assignCreatedTime, sourceVersion;
5784
6585
  let hasChanges, invalidated;
5785
6586
  try {
5786
6587
  updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
5787
6588
  invalidated = metadataFlags & exports.INVALIDATED;
5788
- let version = sourceContext.lastModified || (invalidated && existingVersion);
5789
- hasChanges = invalidated || version > existingVersion || !existingRecord;
5790
- const resolveDuration = performance.now() - start;
6589
+ const reportedVersion = sourceContext.lastModified;
6590
+ const validReportedVersion = typeof reportedVersion === 'number' &&
6591
+ Number.isFinite(reportedVersion) &&
6592
+ reportedVersion > 0 &&
6593
+ reportedVersion <= MAX_DATE_TIMESTAMP;
6594
+ if (validReportedVersion) {
6595
+ // A record version is also this node's ordering token (precedesExistingVersion), so a
6596
+ // source-reported version ahead of local time would make every subsequent local write look
6597
+ // out-of-order and be discarded until wall-clock caught up — freezing the row. Honor what
6598
+ // the source reports, but never beyond now.
6599
+ const versionCeiling = Math.max(sourceTimestamp, Date.now());
6600
+ sourceVersion = Math.min(reportedVersion, versionCeiling);
6601
+ if (sourceVersion !== reportedVersion) {
6602
+ logger_ts_1.logger.trace?.(`Capping future source version for ${tableName} id ${id}: ${reportedVersion} -> ${sourceVersion}`);
6603
+ if (!warnedFutureSourceVersion) {
6604
+ warnedFutureSourceVersion = true;
6605
+ logger_ts_1.logger.warn?.(`The source for ${tableName} reported a lastModified ahead of local time (${new Date(reportedVersion).toISOString()}) for id ${id}; capping cached record versions at local time`);
6606
+ }
6607
+ }
6608
+ }
6609
+ else
6610
+ sourceVersion = sourceTimestamp;
6611
+ hasChanges = invalidated || (validReportedVersion && reportedVersion > existingVersion) || !existingRecord;
6612
+ const resolveDuration = node_perf_hooks_1.performance.now() - start;
5791
6613
  (0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'success');
5792
6614
  if (responseHeaders)
5793
6615
  (0, Headers_ts_1.appendHeader)(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
@@ -5802,7 +6624,7 @@ function makeTable(options) {
5802
6624
  if (status === 304) {
5803
6625
  // revalidation of our current cached record
5804
6626
  updatedRecord = existingRecord;
5805
- version = existingVersion;
6627
+ sourceVersion = existingVersion;
5806
6628
  }
5807
6629
  else if (!CACHEABLE_STATUS_CODES.has(status)) {
5808
6630
  // non-cacheable status - propagate to client without caching
@@ -5880,10 +6702,14 @@ function makeTable(options) {
5880
6702
  if (primaryKey && updatedRecord[primaryKey] !== id)
5881
6703
  updatedRecord[primaryKey] = id;
5882
6704
  }
6705
+ assignCreatedTime = createdTimeProperty && updatedRecord?.[createdTimeProperty.name] == null;
5883
6706
  resolved = true;
6707
+ const resolvedVersion = isRocksDB && updatedRecord && existingVersion != null
6708
+ ? Math.max(sourceVersion, existingVersion)
6709
+ : sourceVersion;
5884
6710
  const resolvedEntry = {
5885
6711
  key: id,
5886
- version,
6712
+ version: resolvedVersion,
5887
6713
  value: updatedRecord,
5888
6714
  expiresAt: sourceContext.expiresAt,
5889
6715
  metadataFlags: 0,
@@ -5922,7 +6748,7 @@ function makeTable(options) {
5922
6748
  }
5923
6749
  else
5924
6750
  reject(error);
5925
- const resolveDuration = performance.now() - start;
6751
+ const resolveDuration = node_perf_hooks_1.performance.now() - start;
5926
6752
  (0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'fail');
5927
6753
  if (responseHeaders)
5928
6754
  (0, Headers_ts_1.appendHeader)(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
@@ -5941,16 +6767,28 @@ function makeTable(options) {
5941
6767
  const sourceWrite = {
5942
6768
  key: id,
5943
6769
  store: primaryStore,
5944
- entry: existingEntry,
6770
+ entry: undefined,
5945
6771
  nodeName: 'source',
5946
- commit: (txnTime, existingEntry, _retry, transaction) => {
6772
+ commit: (_txnTime, existingEntry, _retry, transaction) => {
5947
6773
  sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
5948
- if (existingEntry?.version !== existingVersion) {
5949
- // don't do anything if the version has changed
6774
+ const racedVersion = existingEntry?.version;
6775
+ // A first fill may replace a record that raced it only when its candidate version strictly
6776
+ // orders after that record. The comparison has to be replica-independent, so a tie leaves the
6777
+ // raced record in place: precedesExistingVersion() would break the tie with *this* node's
6778
+ // name, and a fill from a shared source has no node identity of its own, so two replicas
6779
+ // resolving the same tie could keep different values at the same version.
6780
+ const replacesRacedRecord = racedVersion == null || sourceVersion > racedVersion;
6781
+ if (racedVersion !== existingVersion &&
6782
+ // Revalidations retain exact-CAS semantics; first fills use deterministic ordering.
6783
+ (existingVersion != null || !updatedRecord || !replacesRacedRecord)) {
6784
+ logger_ts_1.logger.trace?.(`Discarding resolved record from source with id: ${id}, source version: ${sourceVersion}, current version: ${racedVersion}`);
5950
6785
  sourceWrite.skipped = true;
5951
6786
  return;
5952
6787
  }
5953
- updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
6788
+ const currentRecord = existingEntry?.value;
6789
+ const recordVersion = isRocksDB && racedVersion != null ? Math.max(sourceVersion, racedVersion) : sourceVersion;
6790
+ const txnLogKey = isRocksDB ? transaction?.getTimestamp?.() : recordVersion;
6791
+ updateIndices(id, currentRecord, updatedRecord, transaction && { transaction });
5954
6792
  if (updatedRecord) {
5955
6793
  if (existingEntry) {
5956
6794
  context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
@@ -5961,23 +6799,23 @@ function makeTable(options) {
5961
6799
  if (updatedTimeProperty) {
5962
6800
  updatedRecord[updatedTimeProperty.name] =
5963
6801
  updatedTimeProperty.type === 'Date'
5964
- ? new Date(txnTime)
6802
+ ? new Date(recordVersion)
5965
6803
  : updatedTimeProperty.type === 'String'
5966
- ? new Date(txnTime).toISOString()
5967
- : txnTime;
6804
+ ? new Date(recordVersion).toISOString()
6805
+ : recordVersion;
5968
6806
  }
5969
- if (createdTimeProperty && updatedRecord[createdTimeProperty.name] == null) {
5970
- const existingCreatedTime = existingEntry?.value?.[createdTimeProperty.name];
6807
+ if (assignCreatedTime) {
6808
+ const existingCreatedTime = currentRecord?.[createdTimeProperty.name];
5971
6809
  if (existingCreatedTime != null) {
5972
6810
  updatedRecord[createdTimeProperty.name] = existingCreatedTime;
5973
6811
  }
5974
6812
  else {
5975
6813
  updatedRecord[createdTimeProperty.name] =
5976
6814
  createdTimeProperty.type === 'Date'
5977
- ? new Date(txnTime)
6815
+ ? new Date(recordVersion)
5978
6816
  : createdTimeProperty.type === 'String'
5979
- ? new Date(txnTime).toISOString()
5980
- : txnTime;
6817
+ ? new Date(recordVersion).toISOString()
6818
+ : recordVersion;
5981
6819
  }
5982
6820
  }
5983
6821
  const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
@@ -6010,23 +6848,33 @@ function makeTable(options) {
6010
6848
  }
6011
6849
  residencyId = getResidencyId(residency);
6012
6850
  }
6013
- logger_ts_1.logger.trace?.(`Writing resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`);
6851
+ logger_ts_1.logger.trace?.(`Writing resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`);
6014
6852
  // TODO: We are doing a double check for ifVersion that should probably be cleaned out
6015
- updateRecord(id, updatedRecord, existingEntry, txnTime, omitLocalRecord ? exports.INVALIDATED : 0, (audit && (hasChanges || omitLocalRecord)) || null, {
6853
+ const writeAudit = (audit && (hasChanges || omitLocalRecord)) || null;
6854
+ updateRecord(id, updatedRecord, existingEntry, recordVersion, omitLocalRecord ? exports.INVALIDATED : 0, writeAudit, {
6016
6855
  user: sourceContext?.user,
6017
6856
  expiresAt: sourceContext.expiresAt,
6018
6857
  residencyId,
6019
6858
  transaction,
6020
6859
  tableToTrack: tableName,
6860
+ additionalAuditRefs: writeAudit && txnLogKey !== recordVersion ? [{ version: txnLogKey, nodeId: 0 }] : undefined,
6021
6861
  }, 'put', Boolean(invalidated), auditRecord);
6022
6862
  // arm the eviction scanner, mirroring the .put() path
6023
6863
  if (sourceContext.expiresAt)
6024
6864
  scheduleCleanup();
6025
6865
  }
6026
6866
  else if (existingEntry) {
6027
- logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`);
6867
+ logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`);
6028
6868
  if (audit || trackDeletes) {
6029
- updateRecord(id, null, existingEntry, txnTime, 0, (audit && hasChanges) || null, { user: sourceContext?.user, transaction, tableToTrack: tableName }, 'delete', Boolean(invalidated));
6869
+ updateRecord(id, null, existingEntry, recordVersion, 0, (audit && hasChanges) || null, {
6870
+ user: sourceContext?.user,
6871
+ transaction,
6872
+ tableToTrack: tableName,
6873
+ recordVersion,
6874
+ additionalAuditRefs: audit && hasChanges && txnLogKey !== recordVersion
6875
+ ? [{ version: txnLogKey, nodeId: 0 }]
6876
+ : undefined,
6877
+ }, 'delete', Boolean(invalidated));
6030
6878
  }
6031
6879
  else {
6032
6880
  (0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, existingVersion);
@@ -6186,7 +7034,15 @@ function makeTable(options) {
6186
7034
  },
6187
7035
  };
6188
7036
  }
7037
+ function settlePendingCleanup() {
7038
+ for (const resolve of pendingCleanupResolvers)
7039
+ resolve();
7040
+ pendingCleanupResolvers.clear();
7041
+ }
6189
7042
  function scheduleCleanup(priority) {
7043
+ // a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
7044
+ if (disposed)
7045
+ return;
6190
7046
  let runImmediately = false;
6191
7047
  if (priority) {
6192
7048
  // run immediately if there is a big increase in priority
@@ -6202,9 +7058,19 @@ function makeTable(options) {
6202
7058
  // run on the last thread so we aren't overloading lower-numbered threads
6203
7059
  if (cleanupTimer)
6204
7060
  clearTimeout(cleanupTimer);
6205
- if (!cleanupInterval)
7061
+ if (!cleanupInterval) {
7062
+ // no replacement pass is being scheduled, so nothing is left to settle a superseded one
7063
+ settlePendingCleanup();
6206
7064
  return;
7065
+ }
7066
+ // This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
7067
+ // this pass's scan completes, so a reclamation run is never told the storage was reclaimed
7068
+ // before any scan ran. It has to run now, though — that run blocks its whole path on the
7069
+ // promise, and the replacement's own slot can be a full interval out.
7070
+ if (pendingCleanupResolvers.size > 0)
7071
+ runImmediately = true;
6207
7072
  return new Promise((resolve) => {
7073
+ pendingCleanupResolvers.add(resolve);
6208
7074
  const startOfYear = new Date();
6209
7075
  startOfYear.setMonth(0);
6210
7076
  startOfYear.setDate(1);
@@ -6217,6 +7083,8 @@ function makeTable(options) {
6217
7083
  ? Date.now()
6218
7084
  : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
6219
7085
  const startNextTimer = (nextScheduled) => {
7086
+ if (disposed)
7087
+ return;
6220
7088
  logger_ts_1.logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
6221
7089
  // noinspection JSVoidFunctionReturnValueUsed
6222
7090
  cleanupTimer = setTimeout(() => (lastEvictionCompletion = lastEvictionCompletion.then(async () => {
@@ -6225,8 +7093,11 @@ function makeTable(options) {
6225
7093
  const rootStore = primaryStore.rootStore;
6226
7094
  if (rootStore.status !== 'open') {
6227
7095
  clearTimeout(cleanupTimer);
7096
+ settlePendingCleanup();
6228
7097
  return;
6229
7098
  }
7099
+ // snapshot: an awaiter that arrives during this scan belongs to the pass that supersedes it
7100
+ const settling = [...pendingCleanupResolvers];
6230
7101
  const MAX_CLEANUP_CONCURRENCY = 50;
6231
7102
  const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY);
6232
7103
  let cleanupIndex = 0;
@@ -6306,7 +7177,10 @@ function makeTable(options) {
6306
7177
  catch (error) {
6307
7178
  logger_ts_1.logger.warn?.(`Error in cleanup scan for ${tableName}:`, error);
6308
7179
  }
6309
- resolve(undefined);
7180
+ for (const settle of settling) {
7181
+ pendingCleanupResolvers.delete(settle);
7182
+ settle();
7183
+ }
6310
7184
  cleanupPriority = 0; // reset the priority
6311
7185
  })), Math.min(nextScheduled - Date.now(), hdbTerms_ts_1.MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number
6312
7186
  ).unref(); // don't let this prevent closing the thread
@@ -6317,17 +7191,17 @@ function makeTable(options) {
6317
7191
  }
6318
7192
  function addDeleteRemoval() {
6319
7193
  deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id, version) => {
6320
- primaryStore.remove(id, version);
7194
+ return primaryStore.remove(id, version);
6321
7195
  });
6322
7196
  }
6323
7197
  function runRecordExpirationEviction() {
6324
7198
  // Periodically evict expired records, searching for records who expiresAt timestamp is before now
6325
7199
  if ((0, manageThreads_js_1.getWorkerIndex)() === 0) {
6326
7200
  // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
6327
- setInterval(async () => {
7201
+ recordExpirationInterval = setInterval(async () => {
6328
7202
  // go through each database and table and then search for expired entries
6329
7203
  // find any entries that are set to expire before now
6330
- if (runningRecordExpiration)
7204
+ if (disposed || runningRecordExpiration)
6331
7205
  return;
6332
7206
  runningRecordExpiration = true;
6333
7207
  try {