@harperfast/harper 5.2.13 → 5.3.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (655) hide show
  1. package/agent/mcpTools.ts +1 -1
  2. package/agent/session.ts +25 -14
  3. package/bin/cliOperations.ts +58 -9
  4. package/bin/copyDb.ts +282 -59
  5. package/bin/deploySetup.ts +16 -5
  6. package/bin/harper.ts +1 -1
  7. package/bin/help.ts +4 -1
  8. package/bin/lite.ts +4 -1
  9. package/bin/restart.ts +120 -4
  10. package/bin/run.ts +6 -11
  11. package/bin/upgrade.js +7 -3
  12. package/bin/workloadIdentity.ts +119 -0
  13. package/components/Application.ts +3436 -242
  14. package/components/ApplicationScope.ts +8 -0
  15. package/components/EntryHandler.ts +59 -39
  16. package/components/OptionsWatcher.ts +440 -98
  17. package/components/RuntimeModuleTracker.ts +38 -7
  18. package/components/Scope.ts +56 -15
  19. package/components/awaitRestart.ts +84 -0
  20. package/components/componentLoader.ts +381 -32
  21. package/components/componentPreparationLock.ts +16 -5
  22. package/components/deploymentOperations.ts +4 -1
  23. package/components/deploymentRecorder.ts +9 -2
  24. package/components/mcp/adapters/harperHttp.ts +4 -0
  25. package/components/mcp/listChanged.ts +4 -0
  26. package/components/mcp/toolRegistry.ts +2 -0
  27. package/components/mcp/tools/operations.ts +9 -0
  28. package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
  29. package/components/operations.js +537 -113
  30. package/components/operationsValidation.js +98 -3
  31. package/components/packageComponent.ts +25 -1
  32. package/components/requestRestart.ts +11 -0
  33. package/components/status/ComponentStatusRegistry.ts +59 -0
  34. package/config/RootConfigWatcher.ts +240 -40
  35. package/config/configReadRetry.ts +62 -0
  36. package/config/configUtils.ts +357 -48
  37. package/config/harperConfigEnvVars.ts +170 -27
  38. package/config/parseConfigFile.ts +34 -0
  39. package/config/readConfigFileSync.ts +44 -0
  40. package/config/watcherArming.ts +59 -0
  41. package/config-root.schema.json +33 -0
  42. package/dataLayer/blobBackup.ts +160 -50
  43. package/dataLayer/delete.ts +6 -1
  44. package/dataLayer/harperBridge/ResourceBridge.ts +80 -10
  45. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  46. package/dataLayer/hdbInfoController.ts +34 -1
  47. package/dataLayer/insert.ts +44 -1
  48. package/dataLayer/rocksdbBackup.ts +53 -10
  49. package/dataLayer/schema.ts +11 -1
  50. package/dataLayer/schemaDescribe.ts +8 -1
  51. package/dist/agent/mcpTools.js +1 -1
  52. package/dist/agent/mcpTools.js.map +1 -1
  53. package/dist/agent/session.d.ts +22 -0
  54. package/dist/agent/session.js +26 -15
  55. package/dist/agent/session.js.map +1 -1
  56. package/dist/bin/cliOperations.js +61 -9
  57. package/dist/bin/cliOperations.js.map +1 -1
  58. package/dist/bin/copyDb.d.ts +12 -1
  59. package/dist/bin/copyDb.js +248 -60
  60. package/dist/bin/copyDb.js.map +1 -1
  61. package/dist/bin/deploySetup.d.ts +2 -0
  62. package/dist/bin/deploySetup.js +11 -3
  63. package/dist/bin/deploySetup.js.map +1 -1
  64. package/dist/bin/harper.js +1 -1
  65. package/dist/bin/harper.js.map +1 -1
  66. package/dist/bin/help.js +4 -1
  67. package/dist/bin/help.js.map +1 -1
  68. package/dist/bin/lite.js +4 -1
  69. package/dist/bin/lite.js.map +1 -1
  70. package/dist/bin/restart.js +91 -6
  71. package/dist/bin/restart.js.map +1 -1
  72. package/dist/bin/run.js +4 -10
  73. package/dist/bin/run.js.map +1 -1
  74. package/dist/bin/upgrade.js +4 -3
  75. package/dist/bin/upgrade.js.map +1 -1
  76. package/dist/bin/workloadIdentity.d.ts +18 -0
  77. package/dist/bin/workloadIdentity.js +100 -0
  78. package/dist/bin/workloadIdentity.js.map +1 -0
  79. package/dist/components/Application.d.ts +235 -17
  80. package/dist/components/Application.js +3038 -238
  81. package/dist/components/Application.js.map +1 -1
  82. package/dist/components/ApplicationScope.d.ts +8 -0
  83. package/dist/components/ApplicationScope.js +7 -0
  84. package/dist/components/ApplicationScope.js.map +1 -1
  85. package/dist/components/EntryHandler.js +26 -10
  86. package/dist/components/EntryHandler.js.map +1 -1
  87. package/dist/components/OptionsWatcher.d.ts +4 -0
  88. package/dist/components/OptionsWatcher.js +440 -99
  89. package/dist/components/OptionsWatcher.js.map +1 -1
  90. package/dist/components/RuntimeModuleTracker.js +40 -6
  91. package/dist/components/RuntimeModuleTracker.js.map +1 -1
  92. package/dist/components/Scope.js +52 -13
  93. package/dist/components/Scope.js.map +1 -1
  94. package/dist/components/awaitRestart.d.ts +33 -0
  95. package/dist/components/awaitRestart.js +61 -0
  96. package/dist/components/awaitRestart.js.map +1 -0
  97. package/dist/components/componentLoader.d.ts +38 -1
  98. package/dist/components/componentLoader.js +313 -24
  99. package/dist/components/componentLoader.js.map +1 -1
  100. package/dist/components/componentPreparationLock.d.ts +5 -0
  101. package/dist/components/componentPreparationLock.js +14 -6
  102. package/dist/components/componentPreparationLock.js.map +1 -1
  103. package/dist/components/deploymentOperations.js +4 -1
  104. package/dist/components/deploymentOperations.js.map +1 -1
  105. package/dist/components/deploymentRecorder.d.ts +4 -2
  106. package/dist/components/deploymentRecorder.js +1 -0
  107. package/dist/components/deploymentRecorder.js.map +1 -1
  108. package/dist/components/mcp/adapters/harperHttp.js +4 -0
  109. package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
  110. package/dist/components/mcp/listChanged.js +5 -0
  111. package/dist/components/mcp/listChanged.js.map +1 -1
  112. package/dist/components/mcp/toolRegistry.d.ts +1 -0
  113. package/dist/components/mcp/toolRegistry.js.map +1 -1
  114. package/dist/components/mcp/tools/operations.d.ts +5 -0
  115. package/dist/components/mcp/tools/operations.js +9 -0
  116. package/dist/components/mcp/tools/operations.js.map +1 -1
  117. package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
  118. package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
  119. package/dist/components/operations.d.ts +28 -0
  120. package/dist/components/operations.js +476 -113
  121. package/dist/components/operations.js.map +1 -1
  122. package/dist/components/operationsValidation.js +97 -3
  123. package/dist/components/operationsValidation.js.map +1 -1
  124. package/dist/components/packageComponent.js +24 -0
  125. package/dist/components/packageComponent.js.map +1 -1
  126. package/dist/components/requestRestart.d.ts +1 -0
  127. package/dist/components/requestRestart.js +7 -0
  128. package/dist/components/requestRestart.js.map +1 -1
  129. package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
  130. package/dist/components/status/ComponentStatusRegistry.js +63 -0
  131. package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
  132. package/dist/config/RootConfigWatcher.d.ts +9 -1
  133. package/dist/config/RootConfigWatcher.js +228 -37
  134. package/dist/config/RootConfigWatcher.js.map +1 -1
  135. package/dist/config/configReadRetry.d.ts +8 -0
  136. package/dist/config/configReadRetry.js +62 -0
  137. package/dist/config/configReadRetry.js.map +1 -0
  138. package/dist/config/configUtils.d.ts +11 -2
  139. package/dist/config/configUtils.js +304 -47
  140. package/dist/config/configUtils.js.map +1 -1
  141. package/dist/config/harperConfigEnvVars.d.ts +16 -0
  142. package/dist/config/harperConfigEnvVars.js +162 -25
  143. package/dist/config/harperConfigEnvVars.js.map +1 -1
  144. package/dist/config/parseConfigFile.d.ts +4 -0
  145. package/dist/config/parseConfigFile.js +35 -0
  146. package/dist/config/parseConfigFile.js.map +1 -0
  147. package/dist/config/readConfigFileSync.d.ts +1 -0
  148. package/dist/config/readConfigFileSync.js +47 -0
  149. package/dist/config/readConfigFileSync.js.map +1 -0
  150. package/dist/config/watcherArming.d.ts +15 -0
  151. package/dist/config/watcherArming.js +59 -0
  152. package/dist/config/watcherArming.js.map +1 -0
  153. package/dist/dataLayer/blobBackup.d.ts +49 -20
  154. package/dist/dataLayer/blobBackup.js +139 -50
  155. package/dist/dataLayer/blobBackup.js.map +1 -1
  156. package/dist/dataLayer/delete.js +1 -1
  157. package/dist/dataLayer/delete.js.map +1 -1
  158. package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
  159. package/dist/dataLayer/harperBridge/ResourceBridge.js +72 -12
  160. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  161. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
  162. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  163. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
  164. package/dist/dataLayer/hdbInfoController.d.ts +10 -0
  165. package/dist/dataLayer/hdbInfoController.js +30 -1
  166. package/dist/dataLayer/hdbInfoController.js.map +1 -1
  167. package/dist/dataLayer/insert.d.ts +9 -1
  168. package/dist/dataLayer/insert.js +30 -0
  169. package/dist/dataLayer/insert.js.map +1 -1
  170. package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
  171. package/dist/dataLayer/rocksdbBackup.js +45 -8
  172. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  173. package/dist/dataLayer/schema.js +8 -0
  174. package/dist/dataLayer/schema.js.map +1 -1
  175. package/dist/dataLayer/schemaDescribe.js +8 -1
  176. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  177. package/dist/index.d.ts +2 -1
  178. package/dist/index.js +4 -1
  179. package/dist/index.js.map +1 -1
  180. package/dist/json/systemSchema.json +55 -0
  181. package/dist/resources/DatabaseTransaction.d.ts +77 -0
  182. package/dist/resources/DatabaseTransaction.js +586 -54
  183. package/dist/resources/DatabaseTransaction.js.map +1 -1
  184. package/dist/resources/LMDBTransaction.d.ts +2 -1
  185. package/dist/resources/LMDBTransaction.js +43 -6
  186. package/dist/resources/LMDBTransaction.js.map +1 -1
  187. package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
  188. package/dist/resources/PrimaryRocksDatabase.js +49 -5
  189. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  190. package/dist/resources/RecordEncoder.d.ts +20 -0
  191. package/dist/resources/RecordEncoder.js +110 -9
  192. package/dist/resources/RecordEncoder.js.map +1 -1
  193. package/dist/resources/RequestTarget.d.ts +2 -0
  194. package/dist/resources/RequestTarget.js.map +1 -1
  195. package/dist/resources/Resource.js +117 -24
  196. package/dist/resources/Resource.js.map +1 -1
  197. package/dist/resources/ResourceInterface.d.ts +28 -1
  198. package/dist/resources/ResourceInterface.js.map +1 -1
  199. package/dist/resources/RocksIndexStore.d.ts +6 -1
  200. package/dist/resources/RocksIndexStore.js +24 -9
  201. package/dist/resources/RocksIndexStore.js.map +1 -1
  202. package/dist/resources/RocksTransactionLogStore.d.ts +24 -1
  203. package/dist/resources/RocksTransactionLogStore.js +160 -49
  204. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  205. package/dist/resources/Table.d.ts +165 -10
  206. package/dist/resources/Table.js +1805 -303
  207. package/dist/resources/Table.js.map +1 -1
  208. package/dist/resources/analytics/write.js +10 -3
  209. package/dist/resources/analytics/write.js.map +1 -1
  210. package/dist/resources/auditStore.d.ts +181 -2
  211. package/dist/resources/auditStore.js +640 -29
  212. package/dist/resources/auditStore.js.map +1 -1
  213. package/dist/resources/blob.d.ts +129 -9
  214. package/dist/resources/blob.js +992 -114
  215. package/dist/resources/blob.js.map +1 -1
  216. package/dist/resources/branchDatabase.d.ts +48 -0
  217. package/dist/resources/branchDatabase.js +892 -0
  218. package/dist/resources/branchDatabase.js.map +1 -0
  219. package/dist/resources/crdt.js +50 -12
  220. package/dist/resources/crdt.js.map +1 -1
  221. package/dist/resources/dataLoader.js +3 -4
  222. package/dist/resources/dataLoader.js.map +1 -1
  223. package/dist/resources/databases.d.ts +157 -14
  224. package/dist/resources/databases.js +1462 -325
  225. package/dist/resources/databases.js.map +1 -1
  226. package/dist/resources/defineTable.d.ts +10 -2
  227. package/dist/resources/defineTable.js +9 -1
  228. package/dist/resources/defineTable.js.map +1 -1
  229. package/dist/resources/derivedIndexRegistry.d.ts +5 -0
  230. package/dist/resources/derivedIndexRegistry.js +68 -0
  231. package/dist/resources/derivedIndexRegistry.js.map +1 -0
  232. package/dist/resources/derivedIndexRuntime.d.ts +215 -0
  233. package/dist/resources/derivedIndexRuntime.js +2027 -0
  234. package/dist/resources/derivedIndexRuntime.js.map +1 -0
  235. package/dist/resources/graphql.d.ts +1 -1
  236. package/dist/resources/graphql.js +52 -16
  237. package/dist/resources/graphql.js.map +1 -1
  238. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +103 -9
  239. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +854 -41
  240. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  241. package/dist/resources/indexes/hnswDerivedIndex.d.ts +67 -0
  242. package/dist/resources/indexes/hnswDerivedIndex.js +464 -0
  243. package/dist/resources/indexes/hnswDerivedIndex.js.map +1 -0
  244. package/dist/resources/indexes/hnswPlaneBinding.d.ts +65 -0
  245. package/dist/resources/indexes/hnswPlaneBinding.js +91 -0
  246. package/dist/resources/indexes/hnswPlaneBinding.js.map +1 -0
  247. package/dist/resources/longLivedTransactions.d.ts +71 -0
  248. package/dist/resources/longLivedTransactions.js +358 -0
  249. package/dist/resources/longLivedTransactions.js.map +1 -0
  250. package/dist/resources/models/backendRegistry.d.ts +26 -0
  251. package/dist/resources/models/backendRegistry.js +60 -2
  252. package/dist/resources/models/backendRegistry.js.map +1 -1
  253. package/dist/resources/models/bootstrap.d.ts +33 -1
  254. package/dist/resources/models/bootstrap.js +416 -31
  255. package/dist/resources/models/bootstrap.js.map +1 -1
  256. package/dist/resources/nodeIdMapping.d.ts +5 -0
  257. package/dist/resources/nodeIdMapping.js +49 -0
  258. package/dist/resources/nodeIdMapping.js.map +1 -1
  259. package/dist/resources/recordLock.d.ts +123 -0
  260. package/dist/resources/recordLock.js +315 -0
  261. package/dist/resources/recordLock.js.map +1 -0
  262. package/dist/resources/recordLockCoordinator.d.ts +557 -0
  263. package/dist/resources/recordLockCoordinator.js +2565 -0
  264. package/dist/resources/recordLockCoordinator.js.map +1 -0
  265. package/dist/resources/replayLogs.d.ts +14 -1
  266. package/dist/resources/replayLogs.js +157 -24
  267. package/dist/resources/replayLogs.js.map +1 -1
  268. package/dist/resources/replayLogsGuards.d.ts +94 -7
  269. package/dist/resources/replayLogsGuards.js +111 -7
  270. package/dist/resources/replayLogsGuards.js.map +1 -1
  271. package/dist/resources/replicatedApplyFailure.d.ts +16 -0
  272. package/dist/resources/replicatedApplyFailure.js +63 -0
  273. package/dist/resources/replicatedApplyFailure.js.map +1 -0
  274. package/dist/resources/scheduler/scheduler.js +3 -3
  275. package/dist/resources/scheduler/scheduler.js.map +1 -1
  276. package/dist/resources/search.d.ts +10 -4
  277. package/dist/resources/search.js +283 -37
  278. package/dist/resources/search.js.map +1 -1
  279. package/dist/resources/tracked.d.ts +5 -1
  280. package/dist/resources/tracked.js +74 -23
  281. package/dist/resources/tracked.js.map +1 -1
  282. package/dist/resources/transactionBroadcast.d.ts +1 -1
  283. package/dist/resources/transactionBroadcast.js +2 -2
  284. package/dist/resources/transactionBroadcast.js.map +1 -1
  285. package/dist/security/auth.js +61 -30
  286. package/dist/security/auth.js.map +1 -1
  287. package/dist/security/authn/oidc/claims.d.ts +22 -0
  288. package/dist/security/authn/oidc/claims.js +71 -0
  289. package/dist/security/authn/oidc/claims.js.map +1 -0
  290. package/dist/security/authn/oidc/identityToken.d.ts +27 -0
  291. package/dist/security/authn/oidc/identityToken.js +111 -0
  292. package/dist/security/authn/oidc/identityToken.js.map +1 -0
  293. package/dist/security/authn/oidc/jwks.d.ts +25 -0
  294. package/dist/security/authn/oidc/jwks.js +261 -0
  295. package/dist/security/authn/oidc/jwks.js.map +1 -0
  296. package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
  297. package/dist/security/authn/oidc/providers/generic.js +34 -0
  298. package/dist/security/authn/oidc/providers/generic.js.map +1 -0
  299. package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
  300. package/dist/security/authn/oidc/providers/githubActions.js +129 -0
  301. package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
  302. package/dist/security/authn/oidc/providers/index.d.ts +37 -0
  303. package/dist/security/authn/oidc/providers/index.js +24 -0
  304. package/dist/security/authn/oidc/providers/index.js.map +1 -0
  305. package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
  306. package/dist/security/authn/oidc/tokenExchange.js +306 -0
  307. package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
  308. package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
  309. package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
  310. package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
  311. package/dist/security/authn/oidc/types.d.ts +38 -0
  312. package/dist/security/authn/oidc/types.js +6 -0
  313. package/dist/security/authn/oidc/types.js.map +1 -0
  314. package/dist/security/certificateVerification/index.js +40 -11
  315. package/dist/security/certificateVerification/index.js.map +1 -1
  316. package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
  317. package/dist/security/certificateVerification/trustedIssuers.js +79 -0
  318. package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
  319. package/dist/security/certificateVerification/types.d.ts +1 -0
  320. package/dist/security/credentialProvenance.d.ts +35 -0
  321. package/dist/security/credentialProvenance.js +51 -0
  322. package/dist/security/credentialProvenance.js.map +1 -0
  323. package/dist/security/credentialRejection.d.ts +4 -0
  324. package/dist/security/credentialRejection.js +24 -0
  325. package/dist/security/credentialRejection.js.map +1 -0
  326. package/dist/security/deferredAuthentication.d.ts +36 -0
  327. package/dist/security/deferredAuthentication.js +70 -0
  328. package/dist/security/deferredAuthentication.js.map +1 -0
  329. package/dist/security/impersonation.d.ts +21 -0
  330. package/dist/security/impersonation.js +108 -9
  331. package/dist/security/impersonation.js.map +1 -1
  332. package/dist/security/jsLoader.d.ts +6 -0
  333. package/dist/security/jsLoader.js +77 -15
  334. package/dist/security/jsLoader.js.map +1 -1
  335. package/dist/security/keys.js +301 -71
  336. package/dist/security/keys.js.map +1 -1
  337. package/dist/security/operationScope.d.ts +21 -0
  338. package/dist/security/operationScope.js +36 -0
  339. package/dist/security/operationScope.js.map +1 -0
  340. package/dist/security/permissionsTranslator.js +21 -0
  341. package/dist/security/permissionsTranslator.js.map +1 -1
  342. package/dist/security/tokenAuthentication.d.ts +19 -1
  343. package/dist/security/tokenAuthentication.js +191 -10
  344. package/dist/security/tokenAuthentication.js.map +1 -1
  345. package/dist/security/user.js +4 -3
  346. package/dist/security/user.js.map +1 -1
  347. package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
  348. package/dist/server/DurableSubscriptionsSession.js +67 -10
  349. package/dist/server/DurableSubscriptionsSession.js.map +1 -1
  350. package/dist/server/REST.js +106 -2
  351. package/dist/server/REST.js.map +1 -1
  352. package/dist/server/graphqlQuerying.js +4 -0
  353. package/dist/server/graphqlQuerying.js.map +1 -1
  354. package/dist/server/http.d.ts +16 -1
  355. package/dist/server/http.js +122 -17
  356. package/dist/server/http.js.map +1 -1
  357. package/dist/server/itc/serverHandlers.js +8 -1
  358. package/dist/server/itc/serverHandlers.js.map +1 -1
  359. package/dist/server/jobs/jobProcess.js +6 -2
  360. package/dist/server/jobs/jobProcess.js.map +1 -1
  361. package/dist/server/jobs/jobs.js +4 -1
  362. package/dist/server/jobs/jobs.js.map +1 -1
  363. package/dist/server/liveSubscriptionAuth.d.ts +26 -4
  364. package/dist/server/liveSubscriptionAuth.js +105 -39
  365. package/dist/server/liveSubscriptionAuth.js.map +1 -1
  366. package/dist/server/loadRootComponents.js +49 -10
  367. package/dist/server/loadRootComponents.js.map +1 -1
  368. package/dist/server/mqtt.d.ts +2 -0
  369. package/dist/server/mqtt.js +165 -30
  370. package/dist/server/mqtt.js.map +1 -1
  371. package/dist/server/nodeName.d.ts +2 -0
  372. package/dist/server/nodeName.js +107 -23
  373. package/dist/server/nodeName.js.map +1 -1
  374. package/dist/server/serverHelpers/Headers.d.ts +27 -0
  375. package/dist/server/serverHelpers/Headers.js +145 -1
  376. package/dist/server/serverHelpers/Headers.js.map +1 -1
  377. package/dist/server/serverHelpers/NodeAdapterResponse.d.ts +48 -0
  378. package/dist/server/serverHelpers/NodeAdapterResponse.js +220 -0
  379. package/dist/server/serverHelpers/NodeAdapterResponse.js.map +1 -0
  380. package/dist/server/serverHelpers/Request.d.ts +5 -10
  381. package/dist/server/serverHelpers/Request.js +38 -136
  382. package/dist/server/serverHelpers/Request.js.map +1 -1
  383. package/dist/server/serverHelpers/contentTypes.d.ts +11 -0
  384. package/dist/server/serverHelpers/contentTypes.js +221 -40
  385. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  386. package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
  387. package/dist/server/serverHelpers/deployValidationState.js +9 -19
  388. package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
  389. package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
  390. package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
  391. package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
  392. package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
  393. package/dist/server/serverHelpers/registeredOperations.js +74 -21
  394. package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
  395. package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
  396. package/dist/server/serverHelpers/requestSanitization.js +20 -0
  397. package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
  398. package/dist/server/serverHelpers/serverHandlers.js +6 -3
  399. package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
  400. package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
  401. package/dist/server/serverHelpers/serverUtilities.js +178 -29
  402. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  403. package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
  404. package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
  405. package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
  406. package/dist/server/serverHelpers/uwsServer.js +19 -1
  407. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  408. package/dist/server/static.js +24 -28
  409. package/dist/server/static.js.map +1 -1
  410. package/dist/server/storageReclamation.d.ts +5 -0
  411. package/dist/server/storageReclamation.js +17 -1
  412. package/dist/server/storageReclamation.js.map +1 -1
  413. package/dist/server/threads/isolatedApplications.d.ts +47 -0
  414. package/dist/server/threads/isolatedApplications.js +171 -0
  415. package/dist/server/threads/isolatedApplications.js.map +1 -0
  416. package/dist/server/threads/itc.d.ts +7 -2
  417. package/dist/server/threads/itc.js +5 -1
  418. package/dist/server/threads/itc.js.map +1 -1
  419. package/dist/server/threads/logRotationTransport.d.ts +1 -0
  420. package/dist/server/threads/logRotationTransport.js +33 -0
  421. package/dist/server/threads/logRotationTransport.js.map +1 -0
  422. package/dist/server/threads/manageThreads.d.ts +83 -3
  423. package/dist/server/threads/manageThreads.js +769 -75
  424. package/dist/server/threads/manageThreads.js.map +1 -1
  425. package/dist/server/threads/socketRouter.d.ts +1 -0
  426. package/dist/server/threads/socketRouter.js +277 -31
  427. package/dist/server/threads/socketRouter.js.map +1 -1
  428. package/dist/server/threads/threadHeapMemory.d.ts +2 -0
  429. package/dist/server/threads/threadHeapMemory.js +31 -0
  430. package/dist/server/threads/threadHeapMemory.js.map +1 -0
  431. package/dist/server/threads/threadServer.js +76 -22
  432. package/dist/server/threads/threadServer.js.map +1 -1
  433. package/dist/sqlEngine/config.d.ts +1 -3
  434. package/dist/sqlEngine/config.js +19 -16
  435. package/dist/sqlEngine/config.js.map +1 -1
  436. package/dist/sqlTranslator/index.d.ts +1 -1
  437. package/dist/sqlTranslator/index.js +30 -7
  438. package/dist/sqlTranslator/index.js.map +1 -1
  439. package/dist/upgrade/directives/5-3-0.d.ts +7 -0
  440. package/dist/upgrade/directives/5-3-0.js +148 -0
  441. package/dist/upgrade/directives/5-3-0.js.map +1 -0
  442. package/dist/upgrade/directives/directivesController.js +2 -1
  443. package/dist/upgrade/directives/directivesController.js.map +1 -1
  444. package/dist/utility/OperationFunctionCaller.js +2 -1
  445. package/dist/utility/OperationFunctionCaller.js.map +1 -1
  446. package/dist/utility/common_utils.d.ts +16 -0
  447. package/dist/utility/common_utils.js +32 -6
  448. package/dist/utility/common_utils.js.map +1 -1
  449. package/dist/utility/componentNames.d.ts +8 -0
  450. package/dist/utility/componentNames.js +12 -1
  451. package/dist/utility/componentNames.js.map +1 -1
  452. package/dist/utility/environment/environmentManager.js +3 -6
  453. package/dist/utility/environment/environmentManager.js.map +1 -1
  454. package/dist/utility/environment/systemInformation.d.ts +1 -0
  455. package/dist/utility/environment/systemInformation.js +1 -0
  456. package/dist/utility/environment/systemInformation.js.map +1 -1
  457. package/dist/utility/errors/commonErrors.d.ts +2 -0
  458. package/dist/utility/errors/commonErrors.js +2 -0
  459. package/dist/utility/errors/commonErrors.js.map +1 -1
  460. package/dist/utility/errors/hdbError.d.ts +33 -0
  461. package/dist/utility/errors/hdbError.js +58 -1
  462. package/dist/utility/errors/hdbError.js.map +1 -1
  463. package/dist/utility/globalSchema.d.ts +18 -0
  464. package/dist/utility/hdbTerms.d.ts +18 -0
  465. package/dist/utility/hdbTerms.js +20 -2
  466. package/dist/utility/hdbTerms.js.map +1 -1
  467. package/dist/utility/logging/harper_logger.d.ts +2 -0
  468. package/dist/utility/logging/harper_logger.js +286 -30
  469. package/dist/utility/logging/harper_logger.js.map +1 -1
  470. package/dist/utility/logging/logGenerationCoordinator.d.ts +35 -0
  471. package/dist/utility/logging/logGenerationCoordinator.js +184 -0
  472. package/dist/utility/logging/logGenerationCoordinator.js.map +1 -0
  473. package/dist/utility/logging/logRotation.d.ts +46 -0
  474. package/dist/utility/logging/logRotation.js +365 -0
  475. package/dist/utility/logging/logRotation.js.map +1 -0
  476. package/dist/utility/logging/logRotator.d.ts +1 -1
  477. package/dist/utility/logging/logRotator.js +192 -85
  478. package/dist/utility/logging/logRotator.js.map +1 -1
  479. package/dist/utility/nodeIdentity.d.ts +9 -0
  480. package/dist/utility/nodeIdentity.js +58 -0
  481. package/dist/utility/nodeIdentity.js.map +1 -0
  482. package/dist/utility/npmUtilities.js +11 -7
  483. package/dist/utility/npmUtilities.js.map +1 -1
  484. package/dist/utility/operationPermissions.d.ts +3 -1
  485. package/dist/utility/operationPermissions.js +16 -1
  486. package/dist/utility/operationPermissions.js.map +1 -1
  487. package/dist/utility/operation_authorization.d.ts +10 -7
  488. package/dist/utility/operation_authorization.js +212 -41
  489. package/dist/utility/operation_authorization.js.map +1 -1
  490. package/dist/utility/watchPath.d.ts +29 -0
  491. package/dist/utility/watchPath.js +68 -0
  492. package/dist/utility/watchPath.js.map +1 -0
  493. package/dist/utility/watcherFallback.d.ts +41 -0
  494. package/dist/utility/watcherFallback.js +153 -0
  495. package/dist/utility/watcherFallback.js.map +1 -1
  496. package/dist/validation/configValidator.d.ts +12 -0
  497. package/dist/validation/configValidator.js +205 -75
  498. package/dist/validation/configValidator.js.map +1 -1
  499. package/dist/validation/installValidator.js +12 -0
  500. package/dist/validation/installValidator.js.map +1 -1
  501. package/dist/validation/validationWrapper.d.ts +11 -0
  502. package/dist/validation/validationWrapper.js +16 -3
  503. package/dist/validation/validationWrapper.js.map +1 -1
  504. package/index.ts +8 -0
  505. package/json/systemSchema.json +55 -0
  506. package/npm-shrinkwrap.json +286 -194
  507. package/package.json +15 -7
  508. package/resources/DESIGN.md +219 -52
  509. package/resources/DatabaseTransaction.ts +657 -52
  510. package/resources/LMDBTransaction.ts +46 -6
  511. package/resources/PrimaryRocksDatabase.ts +46 -7
  512. package/resources/RecordEncoder.ts +124 -9
  513. package/resources/RequestTarget.ts +2 -0
  514. package/resources/Resource.ts +114 -22
  515. package/resources/ResourceInterface.ts +31 -0
  516. package/resources/RocksIndexStore.ts +30 -9
  517. package/resources/RocksTransactionLogStore.ts +188 -51
  518. package/resources/Table.ts +1960 -301
  519. package/resources/analytics/write.ts +10 -3
  520. package/resources/auditStore.ts +647 -32
  521. package/resources/blob.ts +1029 -110
  522. package/resources/branchDatabase.ts +941 -0
  523. package/resources/crdt.ts +76 -12
  524. package/resources/dataLoader.ts +3 -4
  525. package/resources/databases.ts +1584 -320
  526. package/resources/defineTable.ts +18 -2
  527. package/resources/derivedIndexRegistry.ts +56 -0
  528. package/resources/derivedIndexRuntime.ts +2292 -0
  529. package/resources/graphql.ts +72 -17
  530. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +911 -48
  531. package/resources/indexes/hnswDerivedIndex.ts +531 -0
  532. package/resources/indexes/hnswPlaneBinding.ts +174 -0
  533. package/resources/longLivedTransactions.ts +360 -0
  534. package/resources/models/backendRegistry.ts +84 -2
  535. package/resources/models/bootstrap.ts +473 -28
  536. package/resources/nodeIdMapping.ts +50 -0
  537. package/resources/recordLock.ts +419 -0
  538. package/resources/recordLockCoordinator.ts +3043 -0
  539. package/resources/replayLogs.ts +152 -26
  540. package/resources/replayLogsGuards.ts +171 -8
  541. package/resources/replicatedApplyFailure.ts +77 -0
  542. package/resources/scheduler/scheduler.ts +4 -4
  543. package/resources/search.ts +288 -46
  544. package/resources/tracked.ts +73 -22
  545. package/resources/transactionBroadcast.ts +3 -3
  546. package/security/auth.ts +68 -29
  547. package/security/authn/oidc/claims.ts +72 -0
  548. package/security/authn/oidc/identityToken.ts +129 -0
  549. package/security/authn/oidc/jwks.ts +260 -0
  550. package/security/authn/oidc/providers/generic.ts +40 -0
  551. package/security/authn/oidc/providers/githubActions.ts +137 -0
  552. package/security/authn/oidc/providers/index.ts +52 -0
  553. package/security/authn/oidc/tokenExchange.ts +300 -0
  554. package/security/authn/oidc/trustPolicyOperations.ts +343 -0
  555. package/security/authn/oidc/types.ts +41 -0
  556. package/security/certificateVerification/index.ts +54 -13
  557. package/security/certificateVerification/trustedIssuers.ts +76 -0
  558. package/security/certificateVerification/types.ts +1 -0
  559. package/security/credentialProvenance.ts +47 -0
  560. package/security/credentialRejection.ts +22 -0
  561. package/security/deferredAuthentication.ts +71 -0
  562. package/security/impersonation.ts +117 -12
  563. package/security/jsLoader.ts +83 -18
  564. package/security/keys.ts +298 -72
  565. package/security/operationScope.ts +33 -0
  566. package/security/permissionsTranslator.js +23 -0
  567. package/security/tokenAuthentication.ts +233 -12
  568. package/security/user.ts +4 -3
  569. package/server/DESIGN.md +194 -16
  570. package/server/DurableSubscriptionsSession.ts +71 -11
  571. package/server/REST.ts +115 -4
  572. package/server/graphqlQuerying.ts +4 -0
  573. package/server/http.ts +133 -20
  574. package/server/itc/serverHandlers.js +8 -1
  575. package/server/jobs/jobProcess.ts +8 -2
  576. package/server/jobs/jobs.ts +4 -1
  577. package/server/liveSubscriptionAuth.ts +129 -46
  578. package/server/loadRootComponents.js +50 -8
  579. package/server/mqtt.ts +179 -38
  580. package/server/nodeName.ts +103 -21
  581. package/server/serverHelpers/Headers.ts +135 -0
  582. package/server/serverHelpers/NodeAdapterResponse.ts +221 -0
  583. package/server/serverHelpers/Request.ts +33 -131
  584. package/server/serverHelpers/contentTypes.ts +217 -36
  585. package/server/serverHelpers/deployValidationState.ts +24 -13
  586. package/server/serverHelpers/operationAuthorizationState.ts +34 -3
  587. package/server/serverHelpers/registeredOperations.ts +79 -22
  588. package/server/serverHelpers/requestSanitization.ts +15 -0
  589. package/server/serverHelpers/serverHandlers.js +6 -3
  590. package/server/serverHelpers/serverUtilities.ts +232 -40
  591. package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
  592. package/server/serverHelpers/uwsServer.ts +17 -2
  593. package/server/static.ts +23 -29
  594. package/server/storageReclamation.ts +15 -2
  595. package/server/threads/isolatedApplications.ts +157 -0
  596. package/server/threads/itc.js +11 -1
  597. package/server/threads/logRotationTransport.ts +40 -0
  598. package/server/threads/manageThreads.js +771 -66
  599. package/server/threads/socketRouter.ts +291 -30
  600. package/server/threads/threadHeapMemory.ts +26 -0
  601. package/server/threads/threadServer.js +73 -22
  602. package/sqlTranslator/index.ts +31 -8
  603. package/studio/web/assets/{Chat-4RrB5134.js → Chat-D3j-1yY1.js} +1 -1
  604. package/studio/web/assets/{FloatingChat-omlNMDcJ.js → FloatingChat-BxJGYcfB.js} +3 -3
  605. package/studio/web/assets/{apiToken-Bke3wvfZ.js → apiToken-CT55oWOe.js} +1 -1
  606. package/studio/web/assets/{applications-DvFDYJgK.js → applications-D9Ct9_vm.js} +1 -1
  607. package/studio/web/assets/{cssMode-C1vRa7zh.js → cssMode-DV8H7VwA.js} +1 -1
  608. package/studio/web/assets/{editor-3XRWEDWX.js → editor-uatc0unt.js} +1 -1
  609. package/studio/web/assets/{html-CTY5tdMr.js → html-Bm6D6paN.js} +1 -1
  610. package/studio/web/assets/{htmlMode-CG1vSD9t.js → htmlMode-CEn7tpLG.js} +1 -1
  611. package/studio/web/assets/{index-Dfpeofdu.js → index-BIXW6Pu4.js} +5 -5
  612. package/studio/web/assets/{index.lazy-B1VOIv-t.js → index.lazy-UI7L-Vrk.js} +1 -1
  613. package/studio/web/assets/{javascript-CJeJzGnI.js → javascript-CJ0G3AFZ.js} +1 -1
  614. package/studio/web/assets/{jsonMode-CYPBwM82.js → jsonMode-DQADAYEa.js} +1 -1
  615. package/studio/web/assets/{languageServices-IKH4GUHl.js → languageServices-CAQJXWcI.js} +1 -1
  616. package/studio/web/assets/{lspLanguageFeatures-BC8_gxKG.js → lspLanguageFeatures-CCQ8P5sY.js} +1 -1
  617. package/studio/web/assets/{notifications-BfKBpYcq.js → notifications-BbxTU6Aw.js} +1 -1
  618. package/studio/web/assets/{notifications-DFbArTfC.js → notifications-Cvb3P1lB.js} +1 -1
  619. package/studio/web/assets/{profile-Q4-T6c-S.js → profile-Yyb7gsvL.js} +1 -1
  620. package/studio/web/assets/{regions-CtkV0xje.js → regions-OgjGHlU5.js} +1 -1
  621. package/studio/web/assets/{register-1ZZuMsiA.js → register-6qwNEOY3.js} +2 -2
  622. package/studio/web/assets/{setComponentFile-ZWLcWv5X.js → setComponentFile-BilDMtgB.js} +1 -1
  623. package/studio/web/assets/{setup-DWprJyJy.js → setup-J6qJ7OIU.js} +2 -2
  624. package/studio/web/assets/{status--aNm8isn.js → status-0RWGcfyD.js} +1 -1
  625. package/studio/web/assets/{toggleHighContrast-Cq_lt3XD.js → toggleHighContrast-BIn-vErT.js} +1 -1
  626. package/studio/web/assets/{tsMode-pjgytARx.js → tsMode-DgUXku4d.js} +1 -1
  627. package/studio/web/assets/{typescript-Cdg0mqUh.js → typescript-C9orXcsM.js} +1 -1
  628. package/studio/web/assets/{useEntityRestURL-RZhaY8Rn.js → useEntityRestURL-BEoXXbUB.js} +1 -1
  629. package/studio/web/assets/{workers-BgoXIqQe.js → workers-JVzSDmgx.js} +1 -1
  630. package/studio/web/assets/{xml-BwSeDMiP.js → xml-Cq-S8S4X.js} +1 -1
  631. package/studio/web/assets/{yaml-DotCUG5l.js → yaml-sfoRdh1M.js} +1 -1
  632. package/studio/web/index.html +1 -1
  633. package/upgrade/directives/5-3-0.ts +132 -0
  634. package/upgrade/directives/directivesController.ts +2 -1
  635. package/utility/OperationFunctionCaller.ts +2 -1
  636. package/utility/common_utils.ts +30 -5
  637. package/utility/componentNames.ts +12 -0
  638. package/utility/environment/environmentManager.ts +3 -7
  639. package/utility/environment/systemInformation.ts +7 -0
  640. package/utility/errors/commonErrors.ts +4 -0
  641. package/utility/errors/hdbError.ts +57 -0
  642. package/utility/hdbTerms.ts +19 -0
  643. package/utility/logging/harper_logger.ts +278 -26
  644. package/utility/logging/logGenerationCoordinator.ts +196 -0
  645. package/utility/logging/logRotation.ts +367 -0
  646. package/utility/logging/logRotator.ts +213 -81
  647. package/utility/nodeIdentity.ts +45 -0
  648. package/utility/npmUtilities.ts +12 -8
  649. package/utility/operationPermissions.ts +18 -1
  650. package/utility/operation_authorization.ts +231 -42
  651. package/utility/watchPath.ts +63 -0
  652. package/utility/watcherFallback.ts +148 -0
  653. package/validation/configValidator.ts +215 -75
  654. package/validation/installValidator.ts +15 -0
  655. package/validation/validationWrapper.ts +18 -4
@@ -14,8 +14,9 @@ import {
14
14
  import { type Database } from 'lmdb';
15
15
  import { Script } from 'node:vm';
16
16
  import { randomUUID } from 'node:crypto';
17
- import { getIndexedValues } from '../utility/lmdb/commonUtility.ts';
18
- import { getThisNodeId, exportIdMapping } from './nodeIdMapping.ts';
17
+ import { performance } from 'node:perf_hooks';
18
+ import { getIndexedValues, getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
19
+ import { getThisNodeId, exportIdMapping, getNodeNameForId } from './nodeIdMapping.ts';
19
20
  import lodash from 'lodash';
20
21
  import { ExtendedIterable, SKIP } from '@harperfast/extended-iterable';
21
22
  import type {
@@ -40,25 +41,42 @@ import {
40
41
  isReleasedTransaction,
41
42
  TRANSACTION_STATE,
42
43
  writeKeyId,
44
+ closeWriteInstance,
45
+ type WriteGeneration,
43
46
  } from './DatabaseTransaction.ts';
47
+ import {
48
+ acquireRecordKey,
49
+ lockAttemptKey,
50
+ lockNotHeldError,
51
+ resolveLockOptions,
52
+ type RecordLockHandle,
53
+ type RecordLockOptions,
54
+ type ResolvedRecordLockOptions,
55
+ } from './recordLock.ts';
56
+ import { getThisNodeName } from '../server/nodeName.ts';
44
57
  import * as envMngr from '../utility/environment/environmentManager.ts';
45
58
  import { addSubscription } from './transactionBroadcast.ts';
46
59
  import {
60
+ DerivedIndexLagError,
47
61
  handleHDBError,
48
62
  ClientError,
49
63
  ServerError,
50
64
  AccessViolation,
51
65
  ValidationError,
66
+ UpdateAttributesLockTimeoutError,
67
+ LockUnavailableError,
52
68
  appendErrorContext,
53
69
  type ValidationIssue,
54
70
  } from '../utility/errors/hdbError.ts';
55
71
  import * as signalling from '../utility/signalling.ts';
56
72
  import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.js';
57
73
  import { databases, table } from './databases.ts';
74
+ import { notifyReplicatedApplyFailure } from './replicatedApplyFailure.ts';
58
75
  import {
59
76
  searchByIndex,
60
77
  findAttribute,
61
78
  estimateCondition,
79
+ estimatedEntryCount,
62
80
  flattenKey,
63
81
  COERCIBLE_OPERATORS,
64
82
  executeConditions,
@@ -66,11 +84,44 @@ import {
66
84
  } from './search.ts';
67
85
  import { logger } from '../utility/logging/logger.ts';
68
86
  import { isStaticResourceInstance } from './staticResourceDispatch.ts';
69
- import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts';
87
+ import {
88
+ Addition,
89
+ assignTrackedAccessors,
90
+ updateAndFreeze,
91
+ hasChanges,
92
+ GenericTrackedObject,
93
+ ASSERT_TRACKED_WRITABLE,
94
+ GET_TRACKED_WRITE_GENERATION,
95
+ } from './tracked.ts';
70
96
  import { transaction, contextStorage } from './transaction.ts';
71
97
  import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary';
72
- import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js';
73
- import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts';
98
+ import {
99
+ getWorkerIndex,
100
+ applicationWorkerIndex,
101
+ ownsStoreMaintenance,
102
+ ownsStoreExpiration,
103
+ runsApplicationCodeSingletons,
104
+ isDedicatedWorker,
105
+ } from '../server/threads/manageThreads.js';
106
+ import {
107
+ HAS_BLOBS,
108
+ LOCAL_ONLY,
109
+ auditRetention,
110
+ removeAuditEntry,
111
+ raiseAuditFloor,
112
+ boundedAuditPruneEnd,
113
+ isLockControlType,
114
+ } from './auditStore.ts';
115
+ import { derivedIndexWriteRejection, hasDerivedIndexRegistration } from './derivedIndexRegistry.ts';
116
+ import {
117
+ decodeLockControlPayload,
118
+ encodeLockControlPayload,
119
+ getClusterLockTransport,
120
+ isClusterLockRequired,
121
+ setLockCoordinatorResolver,
122
+ LockCoordinator,
123
+ type LockControlEntry,
124
+ } from './recordLockCoordinator.ts';
74
125
  import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts';
75
126
  import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts';
76
127
  import {
@@ -79,6 +130,7 @@ import {
79
130
  PENDING_LOCAL_TIME,
80
131
  RecordObject,
81
132
  type Entry,
133
+ type StructureCounts,
82
134
  entryMap,
83
135
  storedFieldsOnly,
84
136
  } from './RecordEncoder.ts';
@@ -87,7 +139,12 @@ import { rebuildUpdateBefore } from './crdt.ts';
87
139
  import { appendHeader } from '../server/serverHelpers/Headers.ts';
88
140
  import fs from 'node:fs';
89
141
  import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts';
90
- import { onStorageReclamation, removeStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts';
142
+ import {
143
+ onStorageReclamation,
144
+ removeStorageReclamation,
145
+ removeStorageReclamationHandler,
146
+ getStorageSpaceStats,
147
+ } from '../server/storageReclamation.ts';
91
148
  import { RequestTarget } from './RequestTarget.ts';
92
149
  import harperLogger from '../utility/logging/harper_logger.ts';
93
150
  import { throttle } from '../server/throttle.ts';
@@ -130,8 +187,14 @@ type MaybePromise<T> = T | Promise<T>;
130
187
 
131
188
  const NULL_WITH_TIMESTAMP = new Uint8Array(9);
132
189
  NULL_WITH_TIMESTAMP[8] = 0xc0; // null
190
+ const sourceWriteTypes = new Set(['put', 'patch', 'delete', 'publish', 'message', 'invalidate', 'relocate']);
191
+ const isSourceWriteType = (type: string) => sourceWriteTypes.has(type);
192
+ const SOURCE_APPLY_POSITION = Symbol('sourceApplyPosition');
133
193
  const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
194
+ const MAX_DATE_TIMESTAMP = 8.64e15;
134
195
  const RECORD_PRUNING_INTERVAL = 60000; // one minute
196
+ const MAX_CONCURRENT_HISTORY_REMOVALS = 10;
197
+ const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
135
198
  // RocksDB-only: number of eviction/tombstone removals coalesced into a single transaction commit.
136
199
  // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
137
200
  // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
@@ -140,6 +203,20 @@ const EVICTION_BATCH_SIZE = 100;
140
203
  // letting an unbounded number of open transactions (and their snapshots) accumulate.
141
204
  const MAX_INFLIGHT_EVICTION_BATCHES = 4;
142
205
  const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
206
+ // Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
207
+ // of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
208
+ // into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
209
+ // truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
210
+ // (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
211
+ const MAX_EXACT_COUNT_SCAN = 1_000_000;
212
+ const MAX_EXACT_COUNT_MS = 1_000;
213
+ // Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
214
+ // not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
215
+ // streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
216
+ const MAX_COUNT_PAGE = 10_000;
217
+ // How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
218
+ // check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
219
+ const COUNT_YIELD_INTERVAL = 2_048;
143
220
  // Smallest forward sample `getRecordCount` will extrapolate a record rate from; below it the scan runs
144
221
  // to completion and reports an exact count.
145
222
  const MIN_ESTIMATOR_SAMPLE = 1_000;
@@ -157,6 +234,76 @@ function usableCount(estimate: any): number {
157
234
  envMngr.initSync();
158
235
  const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
159
236
  const LOCK_TIMEOUT = 10000;
237
+ // This bounds schema-lock acquisition; LOCK_TIMEOUT bounds in-flight record writes during a drop.
238
+ export const UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000;
239
+ const UPDATE_ATTRIBUTES_LOCK = 'update-attributes';
240
+ // Contention is otherwise only visible once it becomes a timeout (harper#2251).
241
+ export const UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = 1000;
242
+ // raw ASCII bytes are ordered-binary's encoding of the string, so this addresses the same native
243
+ // lock as string-keyed tryLock/unlock calls
244
+ const updateAttributesLockKey = Buffer.from(UPDATE_ATTRIBUTES_LOCK);
245
+ const lockWait = new Int32Array(new SharedArrayBuffer(4));
246
+
247
+ /** The wait blocks the event loop, so the locked section must stay synchronous. */
248
+ export function acquireUpdateAttributesLock(
249
+ rootStore: RocksDatabase,
250
+ scopeDescription: string,
251
+ timeout = UPDATE_ATTRIBUTES_LOCK_TIMEOUT
252
+ ) {
253
+ if (rootStore.tryLock(updateAttributesLockKey)) return;
254
+ const startTime = performance.now();
255
+ let waitTime = 1;
256
+ while (!rootStore.tryLock(updateAttributesLockKey)) {
257
+ const elapsed = performance.now() - startTime;
258
+ if (elapsed >= timeout) {
259
+ throw new UpdateAttributesLockTimeoutError(
260
+ `Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`
261
+ );
262
+ }
263
+ if (elapsed >= 2) {
264
+ Atomics.wait(lockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
265
+ if (waitTime < 16) waitTime *= 2;
266
+ }
267
+ }
268
+ const waited = performance.now() - startTime;
269
+ // The caller cannot register its release until we return, so a throw here would leak the lock
270
+ // with no `finally` able to reach it.
271
+ if (waited >= UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
272
+ try {
273
+ logger.warn?.(
274
+ `Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms; this worker's event loop was blocked for that wait, and a holder that runs past ${UPDATE_ATTRIBUTES_LOCK_TIMEOUT}ms fails the update outright`
275
+ );
276
+ } catch {}
277
+ }
278
+
279
+ export function releaseUpdateAttributesLock(rootStore: RocksDatabase) {
280
+ rootStore.unlock(updateAttributesLockKey);
281
+ }
282
+
283
+ export function withUpdateAttributesLock<Callback extends () => unknown>(
284
+ rootStore: RocksDatabase,
285
+ scopeDescription: string,
286
+ callback: Callback & (ReturnType<Callback> extends PromiseLike<unknown> ? never : unknown)
287
+ ): ReturnType<Callback> {
288
+ acquireUpdateAttributesLock(rootStore, scopeDescription);
289
+ try {
290
+ const result = callback();
291
+ if (typeof (result as any)?.then === 'function') {
292
+ Promise.resolve(result).catch((error) =>
293
+ logger.error?.(
294
+ `Async update-attributes callback rejected after its lock was released (${scopeDescription})`,
295
+ error
296
+ )
297
+ );
298
+ throw new TypeError(
299
+ `withUpdateAttributesLock callback must be synchronous (${scopeDescription}); asynchronous work may continue after the lock is released`
300
+ );
301
+ }
302
+ return result as ReturnType<Callback>;
303
+ } finally {
304
+ releaseUpdateAttributesLock(rootStore);
305
+ }
306
+ }
160
307
  // Tolerate a redundant column family drop. Drops are broadcast to every worker
161
308
  // thread and each holds its own handle to the same underlying family, so a
162
309
  // concurrent worker may already have dropped it; the storage engine reports
@@ -395,6 +542,54 @@ function chainKeyForId(id: any): string {
395
542
  return typeof id === 'string' ? 's' + id : 'k' + writeKeyId(id);
396
543
  }
397
544
 
545
+ /** Normalizes a passed `context` argument as `transactional()` does; undefined means fall back to ambient. */
546
+ function contextArgument(context: unknown): any {
547
+ if (!context || isReleasedTransaction(context)) return undefined;
548
+ const resolved = (context as any).getContext?.() || context;
549
+ return resolved instanceof DatabaseTransaction ? { transaction: resolved } : resolved;
550
+ }
551
+
552
+ /** The cluster round never ran for a node-scoped handle, so no peer ever deferred to it. */
553
+ function scopeViolation(
554
+ handle: RecordLockHandle,
555
+ resolved: ResolvedRecordLockOptions,
556
+ databaseName: string
557
+ ): ClientError | undefined {
558
+ if (resolved.scope !== 'cluster' || handle.clusterTsR !== undefined) return undefined;
559
+ // The same predicate lock() fails closed on, not the transport alone: a coalesced caller re-checks
560
+ // this after its wait, and a transport unregistered during that wait leaves the database still
561
+ // clustered while the lookup answers undefined. Only the implicit Phase 0 case falls through.
562
+ if (!resolved.scopeRequested && !isClusterLockRequired(databaseName) && !getClusterLockTransport(databaseName))
563
+ return undefined;
564
+ return new ClientError(
565
+ 'This transaction already holds a node-scoped lock on this record, so a cluster-scoped lock cannot be taken on top of it',
566
+ 409
567
+ );
568
+ }
569
+
570
+ /** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
571
+ function isPlainOptions(value: unknown): boolean {
572
+ return (
573
+ typeof value === 'object' &&
574
+ value !== null &&
575
+ !Array.isArray(value) &&
576
+ !(value instanceof URLSearchParams) &&
577
+ (value as any).id === undefined
578
+ );
579
+ }
580
+
581
+ // Lets a transport push a received control entry straight to the right coordinator without
582
+ // importing Table (which would be a cycle through databases.ts).
583
+ setLockCoordinatorResolver(
584
+ (database: string, tableName: string) => (databases as any)[database]?.[tableName]?.lockCoordinator,
585
+ (database: string, tableName: string) => (databases as any)[database]?.[tableName]?.admittingCoordinator,
586
+ (database: string, tableName: string) => {
587
+ const Table = (databases as any)[database]?.[tableName];
588
+ if (typeof Table?.writeLockControlEntry !== 'function') return undefined;
589
+ return (entry: LockControlEntry) => Table.writeLockControlEntry(entry);
590
+ }
591
+ );
592
+
398
593
  export function makeTable(options) {
399
594
  const {
400
595
  primaryKey,
@@ -413,8 +608,14 @@ export function makeTable(options) {
413
608
  description,
414
609
  hidden,
415
610
  cacheControl,
611
+ isBranch,
416
612
  } = options;
417
613
  let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
614
+ // Set when the TTL exists only on this thread: either application code configured it at runtime, or
615
+ // an isolated application's schema was declared here. Hydrating persisted metadata does not set it:
616
+ // dedicated workers open unrelated shared tables too, whose scan remains owned by the pool.
617
+ let ttlConfiguredByApplication = false;
618
+ let ttlFromLoad = false; // true only around the creation-time call below
418
619
  evictionMs ??= 0;
419
620
  // Eviction without explicit expiration means expiration:0. Apply at construction so
420
621
  // describe_all sees it on every worker, not just ones that ran setTTLExpiration.
@@ -424,7 +625,11 @@ export function makeTable(options) {
424
625
  if (!attributes) attributes = [];
425
626
  if (!properties) properties = projectAttributesToProperties(attributes);
426
627
  const updateRecord = recordUpdater(primaryStore, tableId, auditStore);
628
+ // Created on first cluster-scoped lock() or first arriving control entry, and only while a
629
+ // transport is registered for this database.
630
+ let lockCoordinator: LockCoordinator | undefined;
427
631
  let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
632
+ let warnedFutureSourceVersion = false; // likewise (see getFromSource)
428
633
  let sourceLoad: any; // if a source has a load function (replicator), record it here
429
634
  let hasSourceGet: any;
430
635
  let primaryKeyAttribute: Attribute | undefined;
@@ -454,8 +659,12 @@ export function makeTable(options) {
454
659
  let nonPrefetchSequence = 2;
455
660
  let cleanupInterval = 86400000;
456
661
  let cleanupPriority = 0;
457
- let lastCleanupInterval: number;
662
+ let lastCleanupInterval: number | undefined;
458
663
  let cleanupTimer: NodeJS.Timeout;
664
+ let recordExpirationInterval: NodeJS.Timeout;
665
+ // a reclamation pass awaits a scheduled cleanup, which only settles from its timer
666
+ const pendingCleanupResolvers = new Set<() => void>();
667
+ let disposed = false;
459
668
  // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
460
669
  let expirationScanScheduled = false;
461
670
  // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
@@ -504,9 +713,10 @@ export function makeTable(options) {
504
713
  const MAX_PREFETCH_SEQUENCE = 10;
505
714
  const MAX_PREFETCH_BUNDLE = 6;
506
715
  if (audit) addDeleteRemoval();
507
- onStorageReclamation(primaryStore.path, (priority: number) => {
716
+ const reclamationHandler = (priority: number) => {
508
717
  if (hasSourceGet) return scheduleCleanup(priority);
509
- });
718
+ };
719
+ onStorageReclamation(primaryStore.path, reclamationHandler);
510
720
 
511
721
  class Updatable extends GenericTrackedObject implements RecordObject {
512
722
  declare set: (property: string, value: any) => void;
@@ -597,13 +807,102 @@ export function makeTable(options) {
597
807
  },
598
808
  });
599
809
  }
810
+ function resolveAuditHead(
811
+ id: Id,
812
+ version: number | undefined,
813
+ nodeId: number | undefined,
814
+ refs?: Array<{ version: number; nodeId: number }>
815
+ ) {
816
+ if (!refs?.length) return { txnLogKey: version, nodeId };
817
+ const visited = new Set<string>();
818
+ function findHead(candidateRefs?: Array<{ version: number; nodeId: number }>) {
819
+ if (!candidateRefs) return;
820
+ const pending: Array<{ version: number; nodeId: number }> = candidateRefs.slice().reverse();
821
+ while (pending.length > 0) {
822
+ const ref = pending.pop()!;
823
+ const identity = `${ref.nodeId ?? 0}:${ref.version}`;
824
+ if (visited.has(identity)) continue;
825
+ visited.add(identity);
826
+ const entry = auditStore.getSync(ref.version, tableId, id, ref.nodeId);
827
+ if (!entry) continue;
828
+ if (entry.version === version && (nodeId == null || (entry.nodeId ?? 0) === nodeId))
829
+ return { txnLogKey: ref.version, nodeId: ref.nodeId };
830
+ const previousRefs = entry.previousAdditionalAuditRefs;
831
+ if (previousRefs) {
832
+ for (let index = previousRefs.length - 1; index >= 0; index--) pending.push(previousRefs[index]);
833
+ }
834
+ }
835
+ }
836
+ const referencedHead = findHead(refs);
837
+ if (referencedHead) return referencedHead;
838
+ if (version != null) {
839
+ const directHead = auditStore.getSync(version, tableId, id, nodeId);
840
+ if (directHead?.version === version && (nodeId == null || (directHead.nodeId ?? 0) === nodeId))
841
+ return { txnLogKey: version, nodeId };
842
+ }
843
+ return { txnLogKey: version, nodeId };
844
+ }
845
+ // Canonical-source applies (sourceApply), replay and replication notifications are never shed;
846
+ // dropping one would advance the source cursor past a write that never landed.
847
+ function assertDerivedIndexAdmission(options: any, transaction: any) {
848
+ if (options?.isNotification || transaction?.sourceApply || transaction?.isReplay) return;
849
+ const reason = derivedIndexWriteRejection(auditStore, tableId);
850
+ if (reason) throw new DerivedIndexLagError(reason);
851
+ }
852
+ function stageDerivedIndexEviction(transaction: RocksTransaction, id: Id, version: number) {
853
+ if (!hasDerivedIndexRegistration(auditStore, tableId)) return;
854
+ const nodeId = getThisNodeId(auditStore) ?? 0;
855
+ auditStore.put(
856
+ null,
857
+ {
858
+ type: 'evict',
859
+ tableId,
860
+ recordId: id,
861
+ version,
862
+ nodeId,
863
+ extendedType: LOCAL_ONLY,
864
+ },
865
+ { transaction, nodeId }
866
+ );
867
+ }
600
868
  class TableResource<Record extends object = any> extends Resource<Record> {
601
869
  #record: any; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
602
870
  #changes: any; // the changes to the record that have been made (should not be modified directly)
603
871
  #version?: number; // version of the record
604
872
  #entry?: Entry; // the entry from the database
605
873
  #savingOperation?: any; // operation for the record is currently being saved
874
+ #lockHandle?: RecordLockHandle; // the record lock acquired by lock() — scoped or hold
875
+ #lockWritable?: boolean; // set by #reloadLocked to let save() stage lock-writable updates
876
+ #writeGeneration?: WriteGeneration;
606
877
  declare getProperty: (name: string) => any;
878
+ [ASSERT_TRACKED_WRITABLE](generation = this.#writeGeneration): void {
879
+ if (!generation) return;
880
+ if (generation.internalWrites > 0) return;
881
+ if (generation !== this.#writeGeneration || generation.closed)
882
+ throw new ClientError('Can not modify an update instance after it has been saved; call update() again', 409);
883
+ }
884
+ [GET_TRACKED_WRITE_GENERATION](): WriteGeneration {
885
+ return (this.#writeGeneration ??= { closed: false, internalWrites: 0 });
886
+ }
887
+
888
+ /**
889
+ * Shared guard: if this instance is lock-writable but the handle is gone (expired or
890
+ * released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
891
+ * in addition to the save() path. Every lock-writable instance carries its own handle in
892
+ * #lockHandle (scoped and hold alike), so we never need to search the registry here.
893
+ */
894
+ #assertLiveHandle(id: Id, allowClosed = false): void {
895
+ if (!allowClosed && this.#writeGeneration?.closed && writeKeyId(id) === writeKeyId(this.getId()))
896
+ this[ASSERT_TRACKED_WRITABLE]();
897
+ if (!this.#lockWritable) return;
898
+ const handle = this.#lockHandle!;
899
+ // Off-key writes through the same resource instance are ordinary; only guard the
900
+ // exact key the lock was acquired for.
901
+ if (handle.keyId !== writeKeyId(id)) return;
902
+ if (handle.isExpired()) {
903
+ throw lockNotHeldError(handle);
904
+ }
905
+ }
607
906
  // #section: static-config
608
907
  static name = tableName; // for display/debugging purposes
609
908
  static primaryStore = primaryStore;
@@ -612,6 +911,13 @@ export function makeTable(options) {
612
911
  static tableName = tableName;
613
912
  static tableId = tableId;
614
913
  static indices = indices;
914
+ static derivedIndexRuntime:
915
+ | {
916
+ close(dropping?: boolean): Promise<void>;
917
+ restoreAfterFailedDrop?(): typeof TableResource.derivedIndexRuntime;
918
+ completeDrop?(dropped?: boolean): void;
919
+ }
920
+ | undefined;
615
921
  static audit = audit;
616
922
  static databasePath = databasePath;
617
923
  static databaseName = databaseName;
@@ -698,8 +1004,65 @@ export function makeTable(options) {
698
1004
  (async () => {
699
1005
  let userRoleUpdate = false;
700
1006
  let lastSequenceId;
1007
+ let pendingApplyFailures: Promise<void> | undefined;
1008
+ const reportDroppedWrite = (event, context, error) => {
1009
+ const position =
1010
+ event === context ? context[SOURCE_APPLY_POSITION] : (event.timestamp ?? context[SOURCE_APPLY_POSITION]);
1011
+ const notification = notifyReplicatedApplyFailure(
1012
+ databaseName,
1013
+ {
1014
+ nodeId: event.nodeId ?? context.nodeId,
1015
+ table: event.table ?? context.table,
1016
+ localTime: event.localTime ?? context.localTime,
1017
+ },
1018
+ position,
1019
+ error,
1020
+ tableName
1021
+ );
1022
+ pendingApplyFailures = pendingApplyFailures
1023
+ ? Promise.all([pendingApplyFailures, notification]).then(noop)
1024
+ : notification;
1025
+ return notification;
1026
+ };
1027
+ /** Cluster lock coordination entries (harper#483 Phase 1) describe no record. */
1028
+ const applyLockControlEvent = (event, context) => {
1029
+ const entry = decodeLockControlPayload(event.type, event.value);
1030
+ if (!entry) {
1031
+ logger.warn?.('discarding a malformed record lock control entry from', event.nodeId, event.type);
1032
+ return reportDroppedWrite(event, context, new Error('Malformed record lock control entry'));
1033
+ }
1034
+ const target = event.table ? databases[databaseName]?.[event.table] : TableResource;
1035
+ try {
1036
+ // The audit header's nodeId is the origin, translated on receive and preserved across
1037
+ // relays. The payload's own names are peer-supplied and prove nothing. Rebuild the id
1038
+ // map on a miss rather than waiting out the negative-cache window: a dropped release
1039
+ // leaves the key's home holding its grant until the delegation's own deadline, and
1040
+ // control entries are far too rare to drive the store.
1041
+ //
1042
+ // Inside the guard, not before it: that rebuild reads the audit store, and a throw
1043
+ // there would escape this sink and stall the apply loop for every later entry — the §8
1044
+ // rule that a receive boundary settles its callers and keeps admission closed.
1045
+ const author = getNodeNameForId(auditStore, event.nodeId, true);
1046
+ if (!author) {
1047
+ logger.warn?.('discarding a record lock control entry whose origin node could not be resolved');
1048
+ return reportDroppedWrite(event, context, new Error('Record lock control origin could not be resolved'));
1049
+ }
1050
+ // The coordinator getter fails closed on an unusable node identity. That is right for
1051
+ // an acquire and wrong here: rejecting out of this sink stalls the apply loop for
1052
+ // every later entry rather than dropping one.
1053
+ // `admittingCoordinator`, because `lockCoordinator` answers undefined while a transport
1054
+ // is momentarily unregistered — and this sink runs off the replication stream, not off
1055
+ // that transport. Dropping a peer's clean-handoff release there leaves the home holding
1056
+ // its grant for the delegation's whole deadline.
1057
+ target?.admittingCoordinator?.applyEntry(entry, author, event.timestamp);
1058
+ } catch (error) {
1059
+ logger.warn?.('dropping a record lock control entry: the coordinator is unavailable', error);
1060
+ return reportDroppedWrite(event, context, error);
1061
+ }
1062
+ };
701
1063
  // perform the write of an individual write event
702
1064
  const writeUpdate = async (event, context) => {
1065
+ if (isLockControlType(event.type)) return applyLockControlEvent(event, context);
703
1066
  const value = event.value;
704
1067
  const Table = event.table ? databases[databaseName][event.table] : TableResource;
705
1068
  if (
@@ -719,6 +1082,9 @@ export function makeTable(options) {
719
1082
  ensureLoaded: false,
720
1083
  nodeId: event.nodeId,
721
1084
  viaNodeId: event.viaNodeId,
1085
+ // the origin's record version, stored as-is so every replica holds the version the
1086
+ // origin holds; the transaction's own timestamp stays the origin's log key
1087
+ version: event.version,
722
1088
  // use per-event expiresAt: batched txn context only holds the first event's expiration
723
1089
  expiresAt: event.expiresAt,
724
1090
  // bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log
@@ -728,6 +1094,14 @@ export function makeTable(options) {
728
1094
  async: true,
729
1095
  };
730
1096
  const id = event.id;
1097
+ if (!isSourceWriteType(event.type)) {
1098
+ logger.error?.('Unknown operation', event.type, event.id);
1099
+ const notification = reportDroppedWrite(event, context, new Error('Unknown source operation'));
1100
+ if (event.finished) await event.finished;
1101
+ return notification;
1102
+ }
1103
+ if (Table && event.type === 'put' && value == null && !shouldRevalidateEvents)
1104
+ await reportDroppedWrite(event, context, new Error('Source-applied put has no record content'));
731
1105
  const resource: TableResource = await Table.getResource(id, context, options);
732
1106
  if (event.finished) await event.finished;
733
1107
  switch (event.type) {
@@ -748,13 +1122,18 @@ export function makeTable(options) {
748
1122
  return resource._writeInvalidate(id, value, options);
749
1123
  case 'relocate':
750
1124
  return resource._writeRelocate(id, options);
751
- default:
752
- logger.error?.('Unknown operation', event.type, event.id);
753
1125
  }
754
1126
  };
755
1127
 
756
1128
  /** Keeps the writes to any one key in arrival order; see DESIGN.md (harper#2211). */
757
1129
  const stageWrite = (event, context) => {
1130
+ // A grant must not queue behind whatever the key it names is doing.
1131
+ if (
1132
+ isLockControlType(event.type) ||
1133
+ !isSourceWriteType(event.type) ||
1134
+ (event.type === 'put' && event.value == null && !shouldRevalidateEvents)
1135
+ )
1136
+ return writeUpdate(event, context);
758
1137
  let chainKey: string | undefined;
759
1138
  try {
760
1139
  const Table = event.table ? databases[databaseName][event.table] : TableResource;
@@ -794,14 +1173,18 @@ export function makeTable(options) {
794
1173
  omitCurrent: true,
795
1174
  };
796
1175
  const subscribeOnThisThread = source.subscribeOnThisThread
797
- ? source.subscribeOnThisThread(getWorkerIndex(), subscriptionOptions)
798
- : getWorkerIndex() === 0;
1176
+ ? source.subscribeOnThisThread(applicationWorkerIndex(), subscriptionOptions)
1177
+ : runsApplicationCodeSingletons(); // set up by the defining application's code, so it runs where that code does
799
1178
  const subscription = hasSubscribe && subscribeOnThisThread && (await source.subscribe?.(subscriptionOptions));
800
1179
  if (subscription) {
801
1180
  let txnInProgress;
802
1181
  // we listen for events by iterating through the async iterator provided by the subscription
803
1182
  for await (const event of subscription) {
1183
+ let failureEvent = event;
1184
+ let failurePosition: number | undefined;
1185
+ let applied = false;
804
1186
  try {
1187
+ failurePosition = event?.timestamp;
805
1188
  if (!event || typeof event !== 'object') {
806
1189
  logger.error?.('Bad subscription event', event);
807
1190
  continue;
@@ -809,6 +1192,13 @@ export function makeTable(options) {
809
1192
  const firstWrite = event.type === 'transaction' ? event.writes[0] : event;
810
1193
  if (!firstWrite) {
811
1194
  logger.error?.('Bad subscription event', event);
1195
+ await notifyReplicatedApplyFailure(
1196
+ databaseName,
1197
+ event,
1198
+ failurePosition,
1199
+ new Error('Subscription transaction has no writes'),
1200
+ tableName
1201
+ );
812
1202
  continue;
813
1203
  }
814
1204
  event.source = source;
@@ -817,11 +1207,16 @@ export function makeTable(options) {
817
1207
  // there is no re-subscribe / sequence-id-resume path to recover it. Mark the context so the
818
1208
  // commit retries such conflicts without a cap (see DatabaseTransaction commit).
819
1209
  event.sourceApply = true;
1210
+ event[SOURCE_APPLY_POSITION] = failurePosition;
820
1211
  if (event.type === 'end_txn') {
821
1212
  // Capture the in-progress transaction in a stable local: the loop variable is reset
822
1213
  // once this transaction completes (below), but the seq-id closure and the commit await
823
1214
  // still need to reference it afterward.
824
1215
  const committingTxn = txnInProgress;
1216
+ if (committingTxn) {
1217
+ failureEvent = committingTxn;
1218
+ failurePosition = committingTxn[SOURCE_APPLY_POSITION];
1219
+ }
825
1220
  committingTxn?.resolve();
826
1221
  let updateRecordedSequenceId: () => MaybePromise<void>;
827
1222
  if (event.localTime && lastSequenceId !== event.localTime) {
@@ -911,6 +1306,7 @@ export function makeTable(options) {
911
1306
  let committed;
912
1307
  try {
913
1308
  committed = committingTxn ? await committingTxn.committed : undefined;
1309
+ applied = true;
914
1310
  if (event.onCommit) {
915
1311
  // the onCommit callback can be async and carry associated work (e.g. blob
916
1312
  // transfer); wait for it too before recording the sequence id. Pass the commit
@@ -944,6 +1340,13 @@ export function makeTable(options) {
944
1340
  // than rethrow) so the current beginTxn still starts a fresh transaction with
945
1341
  // correct boundaries instead of having its writes applied as standalone ones.
946
1342
  logger.error?.('source-applied transaction commit failed during apply', error);
1343
+ await notifyReplicatedApplyFailure(
1344
+ databaseName,
1345
+ txnInProgress,
1346
+ txnInProgress[SOURCE_APPLY_POSITION],
1347
+ error,
1348
+ tableName
1349
+ );
947
1350
  } finally {
948
1351
  // Clear it regardless of outcome so a rejected commit isn't re-awaited on the
949
1352
  // next beginTxn (which would brick the apply loop).
@@ -955,7 +1358,9 @@ export function makeTable(options) {
955
1358
  continue;
956
1359
  }
957
1360
  }
958
- // use the version as the transaction timestamp
1361
+ // A source that reports no log position of its own (no `timestamp`) has only one clock,
1362
+ // so its record version doubles as the apply transaction's timestamp. A replication
1363
+ // receiver always sets `timestamp` from the origin's log key and never reaches this.
959
1364
  if (!event.timestamp && event.version) event.timestamp = event.version;
960
1365
  const commitResolution = transaction(event, () => {
961
1366
  if (event.type === 'transaction') {
@@ -1024,6 +1429,7 @@ export function makeTable(options) {
1024
1429
  // standalone write: backpressure on the commit before pulling the next event,
1025
1430
  // and pass the commit resolution through to the callback.
1026
1431
  const committed = commitResolution ? await commitResolution : undefined;
1432
+ applied = true;
1027
1433
  await event.onCommit(committed);
1028
1434
  }
1029
1435
  } else if (commitResolution && !txnInProgress) {
@@ -1032,6 +1438,14 @@ export function makeTable(options) {
1032
1438
  }
1033
1439
  } catch (error) {
1034
1440
  logger.error?.('error in subscription handler', error);
1441
+ if (!applied)
1442
+ await notifyReplicatedApplyFailure(databaseName, failureEvent, failurePosition, error, tableName);
1443
+ } finally {
1444
+ while (pendingApplyFailures) {
1445
+ const notification = pendingApplyFailures;
1446
+ pendingApplyFailures = undefined;
1447
+ await notification;
1448
+ }
1035
1449
  }
1036
1450
  }
1037
1451
  }
@@ -1325,24 +1739,51 @@ export function makeTable(options) {
1325
1739
  * This also informs the scheduling for record eviction.
1326
1740
  * @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`,
1327
1741
  * and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured
1328
- * eviction/scanInterval; object form replaces all three.
1742
+ * eviction/scanInterval; object form replaces all three. An internal schema ownership-only call with
1743
+ * none of those values preserves the settings already loaded from the catalog.
1329
1744
  */
1330
- static setTTLExpiration(opts: number | { expiration?: number; eviction?: number; scanInterval?: number }) {
1745
+ static setTTLExpiration(
1746
+ opts:
1747
+ | number
1748
+ | {
1749
+ expiration?: number;
1750
+ eviction?: number;
1751
+ scanInterval?: number;
1752
+ fromSchema?: boolean;
1753
+ isolatedApplicationOwner?: boolean;
1754
+ }
1755
+ ) {
1331
1756
  if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object'))
1332
1757
  throw new Error('Invalid expiration value type');
1758
+ const declaredHere = typeof opts === 'object' && opts.fromSchema;
1759
+ const isolatedApplicationOwner = declaredHere && opts.isolatedApplicationOwner;
1760
+ const preserveLoadedConfiguration =
1761
+ declaredHere && opts.expiration === undefined && opts.eviction === undefined && opts.scanInterval === undefined;
1762
+ if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlConfiguredByApplication) {
1763
+ ttlConfiguredByApplication = true;
1764
+ // the scan owner may have changed with this: re-evaluate even if the interval did not
1765
+ lastCleanupInterval = undefined;
1766
+ }
1333
1767
  if (typeof opts === 'number') {
1334
1768
  expirationMs = opts * 1000;
1335
- } else {
1769
+ } else if (!preserveLoadedConfiguration) {
1336
1770
  // `??` so an explicit 0 is treated as the user's chosen value, not as "missing"
1337
1771
  expirationMs = (opts.expiration ?? 0) * 1000;
1338
1772
  evictionMs = (opts.eviction ?? 0) * 1000;
1339
1773
  cleanupInterval = (opts.scanInterval ?? 0) * 1000;
1340
1774
  }
1341
1775
  if (expirationMs < 0) throw new Error('Expiration can not be negative');
1342
- // default to one quarter of the total expiration+eviction window
1343
- cleanupInterval = cleanupInterval || (expirationMs + evictionMs) / 4;
1344
- expirationScanScheduled = true;
1345
- scheduleCleanup();
1776
+ if (!preserveLoadedConfiguration) {
1777
+ // default to one quarter of the total expiration+eviction window
1778
+ cleanupInterval = cleanupInterval || (expirationMs + evictionMs) / 4;
1779
+ expirationScanScheduled = true;
1780
+ }
1781
+ // Re-evaluate an existing table-level scan after an ownership-only declaration, but do not
1782
+ // create the default daily cleanup timer for a table that has only an @expiresAt field.
1783
+ if (!preserveLoadedConfiguration || expirationScanScheduled || evictionMs) scheduleCleanup();
1784
+ // @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live
1785
+ // declaration introduces the attribute, including after this application already claimed TTL.
1786
+ if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction();
1346
1787
  }
1347
1788
 
1348
1789
  static getResidencyRecord(id: Id) {
@@ -1429,7 +1870,63 @@ export function makeTable(options) {
1429
1870
  return coerceType(id, primaryKeyAttribute);
1430
1871
  }
1431
1872
 
1873
+ /**
1874
+ * A branch's Table classes deliberately carry the BASE's logical database name so an
1875
+ * application's schema and code resolve unchanged (harper#643). That makes every schema
1876
+ * mutation resolve against the global catalog — a `dropTable()` through a branch would delete
1877
+ * the live base table. Reads and writes are per-branch and unaffected; DDL is refused until a
1878
+ * branch owns a schema identity of its own.
1879
+ */
1880
+ static assertSchemaMutable(operation: string) {
1881
+ if (!isBranch) return;
1882
+ const error: any = new Error(
1883
+ `Cannot ${operation} through a branched database: '${tableName}' resolves to the schema of base ` +
1884
+ `database '${databaseName}', so the change would apply to the base rather than the branch`
1885
+ );
1886
+ error.statusCode = 400;
1887
+ throw error;
1888
+ }
1889
+
1432
1890
  static async dropTable() {
1891
+ TableResource.assertSchemaMutable('drop a table');
1892
+ const rootStore = primaryStore.rootStore;
1893
+ if (
1894
+ databaseName === databasePath &&
1895
+ rootStore instanceof RocksDatabase &&
1896
+ (dbisDb as any).put !== (dbisDb as any).putSync
1897
+ )
1898
+ throw new Error(
1899
+ `Cannot drop ${databaseName}.${TableResource.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`
1900
+ );
1901
+ // Release post-commit derived-index delivery before any destructive work: the runner's
1902
+ // backend must have quiesced before its stores and native file are destroyed, and a
1903
+ // same-name recreate must not race an owner still applying to the old generation.
1904
+ const derivedIndexRuntime = TableResource.derivedIndexRuntime;
1905
+ const restoreDerivedIndexesAfterFailedDrop = () => {
1906
+ try {
1907
+ TableResource.derivedIndexRuntime = derivedIndexRuntime?.restoreAfterFailedDrop?.();
1908
+ } catch (restoreError) {
1909
+ TableResource.derivedIndexRuntime = undefined;
1910
+ logger.error?.(
1911
+ `Could not restore derived indexes after failed drop of ${databaseName}.${TableResource.tableName}`,
1912
+ restoreError
1913
+ );
1914
+ }
1915
+ };
1916
+ try {
1917
+ await derivedIndexRuntime?.close(true);
1918
+ } catch (error) {
1919
+ restoreDerivedIndexesAfterFailedDrop();
1920
+ throw error;
1921
+ }
1922
+ const abortStaleDrop = () => {
1923
+ derivedIndexRuntime?.completeDrop?.(false);
1924
+ TableResource.derivedIndexRuntime = undefined;
1925
+ TableResource.cleanup();
1926
+ if (databases[databaseName]?.[tableName] === TableResource) delete databases[databaseName][tableName];
1927
+ };
1928
+ let dropIdentityConfirmed = databaseName !== databasePath;
1929
+ let primaryCatalogKey = TableResource.tableName + '/';
1433
1930
  if (databaseName === databasePath) {
1434
1931
  // Persist a drop tombstone on the primary catalog entry BEFORE any
1435
1932
  // destructive work. If the process dies or a column family drop fails
@@ -1437,9 +1934,19 @@ export function makeTable(options) {
1437
1934
  // the next startup (or a same-name create) completes the drop via
1438
1935
  // completeInterruptedDrop in databases.ts instead of resurrecting
1439
1936
  // the table.
1440
- const primaryCatalogKey = TableResource.tableName + '/';
1441
- const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey);
1442
- if (primaryMeta && !primaryMeta.dropping) {
1937
+ let tombstoneWrite: any;
1938
+ const writeTombstone = () => {
1939
+ let primaryMeta = (dbisDb as any).getSync(primaryCatalogKey);
1940
+ if (!primaryMeta && primaryKey) {
1941
+ const legacyPrimaryKey = `${TableResource.tableName}/${primaryKey}`;
1942
+ const legacyPrimaryMeta = (dbisDb as any).getSync(legacyPrimaryKey);
1943
+ if (legacyPrimaryMeta?.isPrimaryKey) {
1944
+ primaryCatalogKey = legacyPrimaryKey;
1945
+ primaryMeta = legacyPrimaryMeta;
1946
+ }
1947
+ }
1948
+ if (!primaryMeta || (primaryMeta.tableId != null && primaryMeta.tableId !== tableId)) return false;
1949
+ if (primaryMeta.dropping) return true;
1443
1950
  primaryMeta.dropping = true;
1444
1951
  // Stamps this drop's identity so the interrupted-drop retry budget in
1445
1952
  // databases.ts can be scoped to THIS drop rather than the table name: a
@@ -1449,13 +1956,34 @@ export function makeTable(options) {
1449
1956
  // the budget by generation instead makes the new drop's tombstone carry
1450
1957
  // its own fresh key regardless of what any worker last observed.
1451
1958
  primaryMeta.dropGeneration = randomUUID();
1452
- // put is rebound to putSync on RocksDB stores; on LMDB it returns
1453
- // a promise, so await it to make the tombstone durable before the
1454
- // destructive work below
1455
- const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta);
1456
- if (tombstoneWrite?.then) await tombstoneWrite;
1959
+ tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta);
1960
+ return true;
1961
+ };
1962
+ try {
1963
+ if (rootStore instanceof RocksDatabase) {
1964
+ // withUpdateAttributesLock's locked section cannot be held across an await, so a durable
1965
+ // tombstone depends on put being rebound to putSync for RocksDB primary stores.
1966
+ dropIdentityConfirmed = withUpdateAttributesLock(
1967
+ rootStore,
1968
+ `drop table '${databaseName}.${TableResource.tableName}'`,
1969
+ writeTombstone
1970
+ );
1971
+ } else {
1972
+ rootStore.transactionSync(() => {
1973
+ dropIdentityConfirmed = writeTombstone();
1974
+ });
1975
+ if (typeof tombstoneWrite?.then === 'function') await tombstoneWrite;
1976
+ }
1977
+ } catch (error) {
1978
+ restoreDerivedIndexesAfterFailedDrop();
1979
+ throw error;
1457
1980
  }
1458
1981
  }
1982
+ if (!dropIdentityConfirmed) {
1983
+ abortStaleDrop();
1984
+ return;
1985
+ }
1986
+ TableResource.derivedIndexRuntime = undefined;
1459
1987
  // A get() against a sourcedFrom table resolves to its caller before the resolved
1460
1988
  // record's cache write has committed (see getFromSource) - the write lands "in the
1461
1989
  // background" for latency reasons. Flip this BEFORE removing the table from the
@@ -1468,7 +1996,7 @@ export function makeTable(options) {
1468
1996
  // family drops below. If a drop fails past this point the table stays
1469
1997
  // invisible, and the tombstone guarantees the drop completes on the
1470
1998
  // next startup (or on a same-name create).
1471
- delete databases[databaseName][tableName];
1999
+ if (databases[databaseName]?.[tableName] === TableResource) delete databases[databaseName][tableName];
1472
2000
  // The above stops new source-fill writes from starting, but a write from a get()
1473
2001
  // that already returned to its caller may still be in flight. Dropping the column
1474
2002
  // families out from under that write is a genuine invariant violation, not just a
@@ -1499,15 +2027,21 @@ export function makeTable(options) {
1499
2027
  ]);
1500
2028
  clearTimeout(timer);
1501
2029
  if (result === timedOut) {
2030
+ derivedIndexRuntime?.completeDrop?.();
1502
2031
  throw new Error(
1503
2032
  `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.`
1504
2033
  );
1505
2034
  }
1506
2035
  }
1507
- for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) {
1508
- if (entry.metadataFlags & HAS_BLOBS && entry.value) {
1509
- deleteBlobsInObject(entry.value);
2036
+ try {
2037
+ for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) {
2038
+ if (entry.metadataFlags & HAS_BLOBS && entry.value) {
2039
+ deleteBlobsInObject(entry.value);
2040
+ }
1510
2041
  }
2042
+ } catch (error) {
2043
+ derivedIndexRuntime?.completeDrop?.();
2044
+ throw error;
1511
2045
  }
1512
2046
  if (databaseName === databasePath) {
1513
2047
  // part of a database.
@@ -1525,69 +2059,106 @@ export function makeTable(options) {
1525
2059
  // same-name create completes the interrupted drop and writes fresh
1526
2060
  // catalog rows, and clobbering those would orphan the new table.
1527
2061
  const removeTombstonedCatalog = () => {
1528
- const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/');
1529
- if (!currentPrimary?.dropping) return false;
2062
+ const currentPrimary = (dbisDb as any).getSync(primaryCatalogKey);
2063
+ if (!currentPrimary?.dropping || (currentPrimary.tableId != null && currentPrimary.tableId !== tableId))
2064
+ return false;
1530
2065
  for (const attribute of attributes) {
1531
2066
  dbisDb.remove(TableResource.tableName + '/' + attribute.name);
1532
2067
  }
1533
- dbisDb.remove(TableResource.tableName + '/');
2068
+ dbisDb.remove(primaryCatalogKey);
1534
2069
  return true;
1535
2070
  };
1536
- const rootStore = primaryStore.rootStore;
1537
2071
  if (rootStore instanceof RocksDatabase) {
1538
2072
  // Serialize the drops + catalog removal against a concurrent
1539
2073
  // same-name create (and completeInterruptedDrop) under the database's
1540
2074
  // 'update-attributes' exclusive lock - the same lock the create path
1541
- // holds. It is a synchronous spin lock that blocks the event loop, so
2075
+ // holds. It is a synchronous lock wait that blocks the event loop, so
1542
2076
  // the locked section MUST stay synchronous: drop with dropSync (as
1543
2077
  // completeInterruptedDrop does), never an awaited drop(), or a
1544
- // concurrent create's spin would deadlock waiting on a drop that the
1545
- // blocked event loop can never resolve.
1546
- while (!rootStore.tryLock('update-attributes')) {}
1547
- let removed = false;
2078
+ // concurrent create's wait would be stuck on a drop that the blocked
2079
+ // event loop can never resolve, burning its full deadline before failing.
2080
+ let removed: boolean;
1548
2081
  try {
1549
- for (const attribute of attributes) {
1550
- const index = indices[attribute.name];
1551
- if (index)
1552
- try {
1553
- index.dropSync();
1554
- } catch (error) {
1555
- ignoreAlreadyDropped(error);
1556
- }
1557
- }
1558
- try {
1559
- primaryStore.dropSync();
1560
- } catch (error) {
1561
- ignoreAlreadyDropped(error);
1562
- }
1563
- removed = removeTombstonedCatalog();
1564
- } finally {
1565
- rootStore.unlock('update-attributes');
2082
+ removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
2083
+ const currentPrimary = (dbisDb as any).getSync(primaryCatalogKey);
2084
+ if (!currentPrimary?.dropping || (currentPrimary.tableId != null && currentPrimary.tableId !== tableId))
2085
+ return false;
2086
+ for (const attribute of attributes) {
2087
+ const index = indices[attribute.name];
2088
+ if (index)
2089
+ try {
2090
+ index.customIndex?.resetDerivedStorage?.();
2091
+ index.dropSync();
2092
+ } catch (error) {
2093
+ ignoreAlreadyDropped(error);
2094
+ }
2095
+ }
2096
+ try {
2097
+ primaryStore.dropSync();
2098
+ } catch (error) {
2099
+ ignoreAlreadyDropped(error);
2100
+ }
2101
+ return removeTombstonedCatalog();
2102
+ });
2103
+ if (removed) await dbisDb.committed;
2104
+ } catch (error) {
2105
+ derivedIndexRuntime?.completeDrop?.();
2106
+ throw error;
2107
+ }
2108
+ if (!removed) {
2109
+ abortStaleDrop();
2110
+ return;
1566
2111
  }
1567
- if (removed) await dbisDb.committed;
1568
2112
  } else {
1569
2113
  // LMDB: no shared column-family double-drop, and its engine lock is
1570
2114
  // transactional rather than this spin lock, so keep the awaited drop
1571
2115
  // plus the same tombstone-guarded catalog removal.
1572
- const drops = [];
1573
- for (const attribute of attributes) {
1574
- const index = indices[attribute.name];
1575
- if (index) drops.push(index.drop().catch(ignoreAlreadyDropped));
2116
+ let removed: boolean;
2117
+ try {
2118
+ const currentPrimary = (dbisDb as any).getSync(primaryCatalogKey);
2119
+ if (!currentPrimary?.dropping || (currentPrimary.tableId != null && currentPrimary.tableId !== tableId)) {
2120
+ abortStaleDrop();
2121
+ return;
2122
+ }
2123
+ const drops = [];
2124
+ for (const attribute of attributes) {
2125
+ const index = indices[attribute.name];
2126
+ if (index) {
2127
+ index.customIndex?.resetDerivedStorage?.();
2128
+ drops.push(index.drop().catch(ignoreAlreadyDropped));
2129
+ }
2130
+ }
2131
+ drops.push(primaryStore.drop().catch(ignoreAlreadyDropped));
2132
+ await Promise.all(drops);
2133
+ removed = removeTombstonedCatalog();
2134
+ if (removed) await dbisDb.committed;
2135
+ } catch (error) {
2136
+ derivedIndexRuntime?.completeDrop?.();
2137
+ throw error;
2138
+ }
2139
+ if (!removed) {
2140
+ abortStaleDrop();
2141
+ throw new Error(
2142
+ `Could not complete drop of ${databaseName}.${tableName}: a replacement table became current while the LMDB stores were being dropped`
2143
+ );
1576
2144
  }
1577
- drops.push(primaryStore.drop().catch(ignoreAlreadyDropped));
1578
- await Promise.all(drops);
1579
- if (removeTombstonedCatalog()) await dbisDb.committed;
1580
2145
  }
1581
2146
  } else {
1582
2147
  // legacy table per database. The store to retire is this table's own audit store: nothing
1583
2148
  // assigns `primaryStore.auditStore` — openAuditStore() assigns `rootStore.auditStore`, and
1584
2149
  // this is the reference makeTable() was handed. Awaited so a pass suspended mid-removal has
1585
2150
  // released the primary DBI before it is closed and unlinked.
1586
- await auditStore?.stopAuditCleanup?.();
1587
- removeStorageReclamation(primaryStore.path);
1588
- await primaryStore.close();
1589
- fs.unlinkSync(primaryStore.path);
2151
+ try {
2152
+ await auditStore?.stopAuditCleanup?.();
2153
+ removeStorageReclamation(primaryStore.path);
2154
+ await primaryStore.close();
2155
+ fs.unlinkSync(primaryStore.path);
2156
+ } catch (error) {
2157
+ derivedIndexRuntime?.completeDrop?.();
2158
+ throw error;
2159
+ }
1590
2160
  }
2161
+ derivedIndexRuntime?.completeDrop?.();
1591
2162
  signalling.signalSchemaChange(
1592
2163
  new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName)
1593
2164
  );
@@ -1848,12 +2419,18 @@ export function makeTable(options) {
1848
2419
  } else {
1849
2420
  id = requestTargetToId(target);
1850
2421
  }
2422
+ if (this.#writeGeneration?.closed) {
2423
+ this.#changes = undefined;
2424
+ this.#writeGeneration = undefined;
2425
+ }
2426
+ this.#assertLiveHandle(id, true);
1851
2427
 
1852
2428
  const context = this.getContext();
1853
2429
  const envTxn = txnForContext(context);
1854
2430
  if (!envTxn) throw new Error('Can not update a table resource outside of a transaction');
1855
2431
  // record in the list of updating records so it can be written to the database when we commit
1856
- if (updates === false) {
2432
+ // `false` is the patch-cancel sentinel, not a record root — but only incrementally.
2433
+ if (updates === false && !fullUpdate) {
1857
2434
  // TODO: Remove from transaction
1858
2435
  return this;
1859
2436
  }
@@ -1896,22 +2473,99 @@ export function makeTable(options) {
1896
2473
  });
1897
2474
  }
1898
2475
  }
1899
- return when(this._writeUpdate(id, this.#changes, fullUpdate), () => this);
2476
+ // Keep absent changes distinguishable from an explicit empty patch: framework-created
2477
+ // post/publish updates do not necessarily mutate or save the instance.
2478
+ // A supplied root must reach validation as itself, not as the staged changes (harper#1298).
2479
+ const recordRoot = updates === undefined ? this.#changes : updates;
2480
+ return when(this._writeUpdate(id, recordRoot, fullUpdate), () => this);
1900
2481
  }
1901
2482
 
1902
2483
  /**
1903
2484
  * Save any changes into this instance to the current transaction
1904
2485
  */
1905
2486
  save() {
2487
+ const operation = this.#savingOperation;
2488
+ if (
2489
+ !this.#lockWritable &&
2490
+ this.#writeGeneration?.closed &&
2491
+ (!operation || operation.writeGeneration === this.#writeGeneration)
2492
+ )
2493
+ return;
2494
+ this.#assertLiveHandle(operation?.key ?? this.getId()); // a write through a released or expired lock never lands
2495
+ if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
2496
+ // A held lock's record stages its update here rather than at lock() time: it is often
2497
+ // written after the acquiring transaction has already completed, which would have
2498
+ // dropped an update staged then. Nothing set means nothing to stage — a held-but-untouched
2499
+ // id stays untouched. Scoped locks do not take this branch: #reloadLocked stages their
2500
+ // TransactionWrite at lock() time (exactly like update()), so #savingOperation is always
2501
+ // set for a live scoped lock and the ordinary path below applies.
2502
+ // Verify the hold is still alive: if #lockWritable is set but the handle expired or was
2503
+ // released between lock acquisition and this save(), throw 409 rather than silently
2504
+ // committing stale data. Every lock-writable instance carries its own handle.
2505
+ const saveHandle = this.#lockHandle!;
2506
+ if (saveHandle.isExpired()) {
2507
+ throw lockNotHeldError(saveHandle);
2508
+ }
2509
+ const changes = this.#changes;
2510
+ if (changes && Object.keys(changes).length > 0) {
2511
+ this.#savingOperation = null;
2512
+ return when(this._writeUpdate(this.getId(), changes, false), () => {
2513
+ const op = this.#savingOperation;
2514
+ if (op?.dropped) {
2515
+ this.#changes = undefined;
2516
+ return;
2517
+ }
2518
+ // Clear #savingOperation so the next sequential save() enters the lock-writable
2519
+ // path and creates a fresh write (otherwise a non-null #savingOperation makes
2520
+ // save() take the #saveOperation branch with an already-committed write, which
2521
+ // is a no-op, silently dropping the new change).
2522
+ // op.innerCommit is the real native-transaction commit Promise set on the
2523
+ // immediateCommit path in DatabaseTransaction.save(); await it to ensure
2524
+ // durability before resolving to the caller.
2525
+ if (op?.saved) {
2526
+ this.#savingOperation = null;
2527
+ return op?.innerCommit;
2528
+ }
2529
+ // op.saved = false means addWrite deferred the save; #saveOperation commits it
2530
+ // synchronously but ImmediateTransaction.save() returns undefined while the
2531
+ // inner rocksdb commit is still pending — return innerCommit so the caller
2532
+ // actually waits for durability.
2533
+ return when(this.save(), () => op?.innerCommit);
2534
+ });
2535
+ }
2536
+ // No changes: nothing to stage. A dropped operation (detached at a scoped→hold
2537
+ // upgrade — see detachScopedUpgradeWrite) must not fall through to the ordinary
2538
+ // #saveOperation path below with its now-detached reference.
2539
+ if (!operation || operation.dropped) {
2540
+ this.#savingOperation = null;
2541
+ return;
2542
+ }
2543
+ }
1906
2544
  if (this.#savingOperation) {
2545
+ const operation = this.#savingOperation;
2546
+ this.#savingOperation = null;
2547
+ // A write that lands via a nested immediateCommit (e.g. a second sequential save() on
2548
+ // the same ImmediateTransaction context, once the first has already closed it) sets
2549
+ // operation.innerCommit to the real native-commit promise, but the commit() sweep loop
2550
+ // that triggers it discards its own return value — #saveOperation()'s result can
2551
+ // resolve before that native commit actually settles. Chain on innerCommit (as the
2552
+ // lock-writable hold branch above already does) so callers awaiting save() see the
2553
+ // write durably land, not just the outer (possibly premature) resolution.
2554
+ let result;
1907
2555
  try {
1908
- return this.#saveOperation(this.#savingOperation);
1909
- } finally {
1910
- this.#savingOperation = null;
2556
+ result = this.#saveOperation(operation);
2557
+ } catch (error) {
2558
+ if (!operation.saved) this.#savingOperation = operation;
2559
+ throw error;
1911
2560
  }
2561
+ const innerCommit = operation.innerCommit;
2562
+ return innerCommit ? when(innerCommit, () => result) : result;
1912
2563
  }
1913
2564
  }
1914
2565
  #saveOperation(operation: any) {
2566
+ // LMDB validates staged writes at transaction commit, so bind a lazy update to the
2567
+ // generation selected by save() before another update can replace its changes.
2568
+ operation.captureChanges?.();
1915
2569
  const transaction = txnForContext(this.getContext());
1916
2570
  const holder = operation.stagedIn;
1917
2571
  // never-drop-on-conflict lives on the transaction and would not travel with the write, so an
@@ -1931,13 +2585,26 @@ export function makeTable(options) {
1931
2585
  // merge and index diff would be relative to a record that may never land.
1932
2586
  operation.priorWrite = undefined;
1933
2587
  operation.deferSave = false;
1934
- return when(transaction.addWrite(operation), () => operation.promise ?? operation.result);
2588
+ const result = when(transaction.addWrite(operation), () => operation.promise ?? operation.result);
2589
+ this.#closeWriteChain(operation);
2590
+ return result;
1935
2591
  }
1936
2592
  const owner = holder ?? transaction;
1937
- if (owner.save) return owner.save(operation) || operation.promise || operation.result;
2593
+ if (owner.save) {
2594
+ const result = owner.save(operation) || operation.promise || operation.result;
2595
+ this.#closeWriteChain(operation);
2596
+ return result;
2597
+ }
2598
+ }
2599
+ #closeWriteChain(operation: any) {
2600
+ const owner = operation.stagedIn;
2601
+ for (let write = operation; write && !write.instanceClosed; write = write.priorWrite) {
2602
+ if (write === operation || owner?.ownedWrites?.has(write)) closeWriteInstance(write);
2603
+ }
1938
2604
  }
1939
2605
 
1940
2606
  addTo(property: any, value: any) {
2607
+ this[ASSERT_TRACKED_WRITABLE]();
1941
2608
  if (typeof value === 'number' || typeof value === 'bigint') {
1942
2609
  if (this.#savingOperation?.fullUpdate)
1943
2610
  (this as any).set(property, (+this.getProperty(property) || 0) + (value as any));
@@ -1987,17 +2654,24 @@ export function makeTable(options) {
1987
2654
  });
1988
2655
  }
1989
2656
  _writeInvalidate(id: Id, partialRecord?: any, options?: any) {
2657
+ this.#assertLiveHandle(id);
1990
2658
  const context = this.getContext();
1991
2659
  checkValidId(id);
1992
2660
  const transaction = txnForContext(this.getContext());
2661
+ assertDerivedIndexAdmission(options, transaction);
1993
2662
  const write: any = {
1994
2663
  key: id,
1995
2664
  store: primaryStore,
1996
2665
  invalidated: true,
1997
2666
  entry: this.#entry,
2667
+ recordVersion: options?.version,
2668
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
2669
+ reloadCommitBase: true,
1998
2670
  commit: (txnTime, existingEntry, _retry, transaction: any) => {
2671
+ const txnLogKey =
2672
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
1999
2673
  write.skipped = false; // reset on each retry; cleanup happens after commit if still true
2000
- if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) {
2674
+ if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
2001
2675
  write.skipped = true;
2002
2676
  return;
2003
2677
  }
@@ -2024,9 +2698,15 @@ export function makeTable(options) {
2024
2698
  viaNodeId: options?.viaNodeId,
2025
2699
  transaction,
2026
2700
  tableToTrack: tableName,
2701
+ recordVersion: txnTime,
2702
+ additionalAuditRefs:
2703
+ isRocksDB && audit && txnLogKey !== txnTime
2704
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
2705
+ : undefined,
2027
2706
  },
2028
2707
  'invalidate'
2029
2708
  );
2709
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
2030
2710
  // TODO: recordDeletion?
2031
2711
  },
2032
2712
  };
@@ -2034,20 +2714,27 @@ export function makeTable(options) {
2034
2714
  transaction.addWrite(write);
2035
2715
  }
2036
2716
  _writeRelocate(id: Id, options: any) {
2717
+ this.#assertLiveHandle(id);
2037
2718
  const context = this.getContext();
2038
2719
  checkValidId(id);
2039
2720
  const transaction = txnForContext(this.getContext());
2040
- transaction.addWrite({
2721
+ assertDerivedIndexAdmission(options, transaction);
2722
+ const write: any = {
2041
2723
  key: id,
2042
2724
  store: primaryStore,
2043
2725
  invalidated: true,
2044
2726
  entry: this.#entry,
2727
+ recordVersion: options?.version,
2728
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
2729
+ reloadCommitBase: true,
2045
2730
  before:
2046
2731
  (this.constructor as any).source?.relocate && !(context as any)?.source
2047
2732
  ? (this.constructor as any).source.relocate.bind((this.constructor as any).source, id, undefined, context)
2048
2733
  : undefined,
2049
2734
  commit: (txnTime, existingEntry, _retry, transaction: any) => {
2050
- if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return;
2735
+ const txnLogKey =
2736
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2737
+ if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) return;
2051
2738
  const residency = TableResource.getResidencyRecord(options.residencyId);
2052
2739
  let metadata = 0;
2053
2740
  let newRecord = null;
@@ -2079,13 +2766,20 @@ export function makeTable(options) {
2079
2766
  viaNodeId: options?.viaNodeId,
2080
2767
  expiresAt: options.expiresAt,
2081
2768
  transaction,
2769
+ recordVersion: txnTime,
2770
+ additionalAuditRefs:
2771
+ isRocksDB && audit && txnLogKey !== txnTime
2772
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
2773
+ : undefined,
2082
2774
  },
2083
2775
  'relocate',
2084
2776
  false,
2085
2777
  null
2086
2778
  );
2779
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
2087
2780
  },
2088
- });
2781
+ };
2782
+ transaction.addWrite(write);
2089
2783
  }
2090
2784
 
2091
2785
  /**
@@ -2141,9 +2835,8 @@ export function makeTable(options) {
2141
2835
  // if there is a resolution in-progress, abandon the eviction
2142
2836
  if (primaryStore.hasLock(id, entry.version)) return;
2143
2837
  }
2144
- // evictions never go in the audit log, so we can not record a deletion entry for the eviction
2145
- // as there is no corresponding audit entry and it would never get cleaned up. So we must simply
2146
- // removed the entry entirely, but first cleanup indices
2838
+ // Eviction is not a canonical delete. Indexed caching tables add a local-only control entry so
2839
+ // their derived indexes can remove the resident projection without exposing a delete event.
2147
2840
  let lmdbCompletion: MaybePromise<unknown>;
2148
2841
  if (primaryStore.ifVersion) {
2149
2842
  // lmdb: the index cleanup and the record removal are both version-guarded optimistic writes.
@@ -2156,6 +2849,7 @@ export function makeTable(options) {
2156
2849
  lmdbCompletion = Promise.all([indexCleanup, removal]);
2157
2850
  } else {
2158
2851
  updateIndices(id, existingRecord, null, options);
2852
+ stageDerivedIndexEviction(transaction as RocksTransaction, id, existingVersion);
2159
2853
  removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
2160
2854
  }
2161
2855
  committed = true;
@@ -2201,10 +2895,380 @@ export function makeTable(options) {
2201
2895
  }
2202
2896
  }
2203
2897
  /**
2204
- * This is intended to acquire a lock on a record from the whole cluster.
2898
+ * Static entry point: `Table.lock(id, options?, context?)` — creates an instance in the given,
2899
+ * ambient, or a fresh context and delegates to the instance lock(). This shadows Resource.static
2900
+ * lock so that both callers share the same transaction link (required for cross-instance upgrade
2901
+ * detection). lock() is an in-process API with no authorization hook of its own; it is not
2902
+ * protocol-dispatched, so no allowUpdate/allowCreate check runs on acquisition.
2903
+ *
2904
+ * Dropping the trailing `context` leaks the key: the bare `{}` fallback is an
2905
+ * ImmediateTransaction, which releases no record locks.
2205
2906
  */
2206
- lock() {
2207
- throw new Error('Not yet implemented');
2907
+ static async lock(
2908
+ target?: RequestTargetOrId | RecordLockOptions,
2909
+ options?: RecordLockOptions,
2910
+ context?: any
2911
+ ): Promise<any> {
2912
+ if (!isRocksDB) throw new ClientError('Record locks are not supported on LMDB', 501);
2913
+ if (options === undefined && isPlainOptions(target)) {
2914
+ options = target as RecordLockOptions;
2915
+ target = undefined;
2916
+ }
2917
+ const id = target != null ? requestTargetToId(target as RequestTargetOrId) : null;
2918
+ const resolvedContext: any = contextArgument(context) ?? contextStorage.getStore() ?? {};
2919
+ const resource = new TableResource(id, resolvedContext);
2920
+ return resource.lock(target, options);
2921
+ }
2922
+ /**
2923
+ * Acquire an exclusive lock on this record (or on `target`'s) and return it ready for updates
2924
+ * (harper#483, Phase 0: exclusive across every worker thread of this node). The lock is held
2925
+ * in process memory only — no durable writes. Phase 0 contract: lock() is mutually exclusive
2926
+ * with other lock() calls on the same key; plain writes (put/patch/delete/create) are never
2927
+ * gated or blocked. The generation expires after `lease` if it is never released.
2928
+ *
2929
+ * Transaction-scoped (default): write through the returned record (or the table's static verbs
2930
+ * in the same transaction), and the commit or abort releases it. `{ hold: true }`: the lock
2931
+ * outlives the transaction; write through the returned record and release with `unlock()`, or
2932
+ * let the lease expire.
2933
+ */
2934
+ // async so option/id validation rejects rather than throwing past a caller's `.catch()`; the
2935
+ // body still runs to completion synchronously, which is what keeps concurrent lock() calls
2936
+ // on one key coalescing instead of racing to tryLock.
2937
+ async lock(target?: RequestTargetOrId | RecordLockOptions, options?: RecordLockOptions): Promise<any> {
2938
+ if (!isRocksDB) throw new ClientError('Record locks are not supported on LMDB', 501);
2939
+ if (options === undefined && isPlainOptions(target)) {
2940
+ options = target as RecordLockOptions;
2941
+ target = undefined;
2942
+ }
2943
+ const id = target != null ? requestTargetToId(target as RequestTargetOrId) : this.getId();
2944
+ checkValidId(id);
2945
+ this.#assertLiveHandle(id);
2946
+ const resolved = resolveLockOptions(options);
2947
+ const context = this.getContext();
2948
+ const link = txnForContext(context);
2949
+ const keyId = writeKeyId(id);
2950
+ // Before the re-entrant paths, not after: a transaction that already holds this key
2951
+ // node-scoped would otherwise be handed that handle back for an explicit cluster request,
2952
+ // while the same request on a fresh key fails closed.
2953
+ if (
2954
+ resolved.scope === 'cluster' &&
2955
+ (resolved.scopeRequested || isClusterLockRequired(databaseName)) &&
2956
+ !getClusterLockTransport(databaseName)
2957
+ )
2958
+ return Promise.reject(
2959
+ new LockUnavailableError(
2960
+ `Cluster-scoped record locks are not available on ${databaseName}: no record lock transport is registered`
2961
+ )
2962
+ );
2963
+ const held = this.#lockHandle;
2964
+ if (held && !held.isExpired() && held.keyId === keyId) {
2965
+ // Re-entrant: upgrade to hold if requested, then preserve staged changes.
2966
+ const violation = scopeViolation(held, resolved, databaseName);
2967
+ if (violation) return Promise.reject(violation);
2968
+ if (resolved.hold && !held.hold) {
2969
+ held.upgradeToHold(resolved.lease);
2970
+ // The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
2971
+ // staging is deferred and explicit-save-only, so an unsaved scoped write left
2972
+ // dangling here would otherwise auto-commit at the transaction sweep and clobber
2973
+ // whatever the hold write lands. detachScopedUpgradeWrite marks it .dropped so a
2974
+ // later save() on this instance falls through to the hold branch instead of the
2975
+ // dead #savingOperation reference.
2976
+ detachScopedUpgradeWrite(link, keyId, held);
2977
+ }
2978
+ return Promise.resolve(this.#reloadLocked(id, undefined, true));
2979
+ }
2980
+ const scoped = link.recordLockFor(primaryStore, keyId);
2981
+ if (scoped && !scoped.isExpired()) {
2982
+ const violation = scopeViolation(scoped, resolved, databaseName);
2983
+ if (violation) return Promise.reject(violation);
2984
+ if (resolved.hold && !scoped.hold) {
2985
+ // Upgrade scoped → hold: flip the existing handle object to hold mode so every
2986
+ // instance that already references this handle stays valid. Retiring and creating a
2987
+ // new handle would invalidate those other references (their save() would then throw
2988
+ // 409 against a released handle). The native key stays locked throughout.
2989
+ scoped.upgradeToHold(resolved.lease);
2990
+ detachScopedUpgradeWrite(link, keyId, scoped);
2991
+ return Promise.resolve(this.#reloadLocked(id, scoped, true));
2992
+ }
2993
+ // Already held with the same type: re-entrant return. Preserve any staged changes.
2994
+ return Promise.resolve(this.#reloadLocked(id, scoped, true));
2995
+ }
2996
+ // Cluster scope needs a registered transport. An EXPLICIT { scope: 'cluster' } without one is
2997
+ // a caller asking for a guarantee this node cannot make, so it fails closed rather than
2998
+ // silently returning the node-local lock; the default keeps Phase 0 behavior, which is what
2999
+ // a build with no replication has anyway.
3000
+ // The getter fails closed on an unusable node identity, and lock() answers with a promise.
3001
+ let coordinator: LockCoordinator | undefined;
3002
+ try {
3003
+ coordinator = resolved.scope === 'node' ? undefined : TableResource.lockCoordinator;
3004
+ } catch (error) {
3005
+ return Promise.reject(error as Error);
3006
+ }
3007
+ const key = lockAttemptKey(tableId, id);
3008
+ // Coalesce concurrent lock() calls for the same key inside one link so they don't
3009
+ // self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
3010
+ // reach tryLock before either registers, making the second park against the first.
3011
+ const pending = link.pendingLockFor(primaryStore, keyId);
3012
+ if (pending) {
3013
+ // Wait for the in-flight acquisition, then take the re-entrant path as if
3014
+ // recordLockFor had found it. If the first attempt timed out, re-enter so
3015
+ // the second caller gets its own timeout.
3016
+ // The follower waits on the leader's acquisition, but only for its own timeout.
3017
+ let followerTimer: ReturnType<typeof setTimeout> | undefined;
3018
+ const followerTimedOut = Symbol('follower timeout');
3019
+ // Why the leader failed, so the follower can report that instead of inventing contention
3020
+ // when its own budget runs out. A leader 503 means the guarantee could not be established
3021
+ // at all; retrying is still right (the condition may clear) but 423 at the end is not.
3022
+ let leaderFailure: Error | undefined;
3023
+ const followerStart = Date.now();
3024
+ const followerDeadline = new Promise<never>((_, reject) => {
3025
+ followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
3026
+ });
3027
+ // Try again on this caller's own terms with the budget it has left.
3028
+ const retryOnRemainingBudget = () => {
3029
+ // The enclosing transaction ended while we were parked. A retry re-resolves the
3030
+ // context, which no longer points at this link, so the handle it acquired would be
3031
+ // registered on a fresh transaction that no commit or abort ever releases — the
3032
+ // same abandonment the leader's own post-acquisition guard below rejects.
3033
+ if (link.open === TRANSACTION_STATE.CLOSED && !link.saveCommits)
3034
+ throw new ServerError('Transaction was closed while waiting for a record lock', 500);
3035
+ const remaining = resolved.timeout - (Date.now() - followerStart);
3036
+ if (remaining <= 0)
3037
+ throw leaderFailure ?? new ClientError(`Record is locked and was not released in time`, 423);
3038
+ // Carry the scope only if the caller named it: spreading the resolved options would turn
3039
+ // a defaulted 'cluster' into an explicit one, which is fail-closed when no transport is
3040
+ // registered.
3041
+ return this.lock(target, {
3042
+ lease: resolved.lease,
3043
+ timeout: remaining,
3044
+ hold: resolved.hold,
3045
+ scope: resolved.scopeRequested ? resolved.scope : undefined,
3046
+ }) as Promise<any>;
3047
+ };
3048
+ return Promise.race([pending, followerDeadline]).then(
3049
+ () => {
3050
+ clearTimeout(followerTimer);
3051
+ const acquired = link.recordLockFor(primaryStore, keyId);
3052
+ if (acquired && !acquired.isExpired()) {
3053
+ const violation = scopeViolation(acquired, resolved, databaseName);
3054
+ if (violation) throw violation;
3055
+ if (resolved.hold && !acquired.hold) {
3056
+ detachScopedUpgradeWrite(link, keyId, acquired);
3057
+ acquired.upgradeToHold(resolved.lease);
3058
+ }
3059
+ return this.#reloadLocked(id, acquired, true);
3060
+ }
3061
+ return retryOnRemainingBudget();
3062
+ },
3063
+ (error) => {
3064
+ clearTimeout(followerTimer);
3065
+ // A follower that simply ran out of its own wait was waiting on another caller in this
3066
+ // process, which is the contention 423 describes. But if the LEADER failed for a reason
3067
+ // that is not contention, that reason is the true one — keep it and report it if the
3068
+ // retries below also run out, rather than ending on a 423 for a key nobody held.
3069
+ if (error === followerTimedOut) throw new ClientError(`Record is locked and was not released in time`, 423);
3070
+ if (error instanceof LockUnavailableError) leaderFailure = error;
3071
+ return retryOnRemainingBudget();
3072
+ }
3073
+ );
3074
+ }
3075
+ const pendingPromise = acquireRecordKey(
3076
+ link,
3077
+ primaryStore,
3078
+ key,
3079
+ keyId,
3080
+ resolved.timeout,
3081
+ resolved.lease,
3082
+ resolved.hold
3083
+ );
3084
+ const clusterStart = performance.now();
3085
+ // What a follower waits on must span the cluster round and registration, not just the native
3086
+ // acquire. Waking it at the native hand-off leaves it in a window where the key is held but no
3087
+ // handle is registered, so it retries and parks on the leader's own lock for its full timeout
3088
+ // — inside a transaction that cannot finish until it gives up.
3089
+ const acquisition = pendingPromise.then(async (handle) => {
3090
+ const closedWhileWaiting = () => link.open === TRANSACTION_STATE.CLOSED && !link.saveCommits;
3091
+ if (closedWhileWaiting()) {
3092
+ // The transaction was aborted while this call waited; nothing would ever release the handle.
3093
+ handle.release();
3094
+ throw new ServerError('Transaction was closed while waiting for a record lock', 500);
3095
+ }
3096
+ // Anything that fails from here must give the native key back, or it becomes a lock this
3097
+ // caller does not know it owns.
3098
+ // Re-resolved, not the snapshot taken before `acquireRecordKey`: that wait can run the
3099
+ // caller's whole timeout, long enough for harper-pro to register the transport on this
3100
+ // worker. Using the snapshot would take the native key alone and hand back a node-scoped
3101
+ // handle while a peer that already had the transport is granted the same key.
3102
+ try {
3103
+ if (resolved.scope !== 'node') coordinator = TableResource.lockCoordinator ?? coordinator;
3104
+ } catch (error) {
3105
+ // The getter fails closed on an unusable node identity, and that has to reach the caller
3106
+ // the same way it does before the wait. Swallowing it let an implicit cluster lock fall
3107
+ // through to node-local authority — the one outcome failing closed exists to prevent —
3108
+ // because `coordinator` is still whatever it was, including undefined.
3109
+ handle.release();
3110
+ throw error as Error;
3111
+ }
3112
+ if (coordinator) {
3113
+ try {
3114
+ // Not a 423 when the budget is gone, and not a skip either: the native wait can consume
3115
+ // the whole timeout, and `acquire` with no wait left still admits from a live delegation
3116
+ // or a local grant without sending anything. Only if it cannot does the caller learn the
3117
+ // guarantee was unavailable — which is not the same as the key being held.
3118
+ const remaining = Math.max(0, resolved.timeout - (performance.now() - clusterStart));
3119
+ const round = await coordinator.acquire(id, resolved.lease, remaining);
3120
+ // Resolved through the getter rather than captured, so a transport swap between
3121
+ // acquisition and release reaches the coordinator that now owns the delegation.
3122
+ if (
3123
+ !handle.joinClusterRound(round.tsR, resolved.lease, round.mintedMono, () =>
3124
+ TableResource.admittingCoordinator?.release(id, round.admissionId)
3125
+ )
3126
+ ) {
3127
+ // The round completed inside its lease but the lease elapsed before the handle
3128
+ // could take it. The coordinator still holds it, and only this call knows the
3129
+ // hold was never handed out.
3130
+ // The getter, not the captured coordinator: after a transport swap the captured one no
3131
+ // longer owns this admission, so releasing through it would be a silent no-op.
3132
+ // `.then`, not `Promise.resolve(release())`: the call can throw synchronously, and that
3133
+ // throw would escape the catch and replace the 423 below with an internal error.
3134
+ Promise.resolve()
3135
+ .then(() => TableResource.admittingCoordinator?.release(id, round.admissionId))
3136
+ .catch(noop);
3137
+ // 503, not 423: the home granted this key to US and the lease elapsed before the handle
3138
+ // could take it, so nobody ever held it. The coordinator classifies the same thing the
3139
+ // same way — see its `timeout` denial.
3140
+ throw new LockUnavailableError(
3141
+ `A cluster record lock on ${databaseName}.${tableName} was granted after its lease had elapsed`
3142
+ );
3143
+ }
3144
+ // A recall must be able to fence a write this handle staged and then unlocked, so
3145
+ // the coordinator needs a way to revoke it — see LockCoordinator.registerAdmission.
3146
+ // The getter again: a swap during the acquisition moved this admission to the
3147
+ // successor, and registering on the predecessor would revoke a handle that is fine.
3148
+ TableResource.admittingCoordinator?.registerAdmission(round.admissionId, () => handle.revokeLease());
3149
+ } catch (error) {
3150
+ handle.release();
3151
+ throw error;
3152
+ }
3153
+ if (closedWhileWaiting()) {
3154
+ handle.release();
3155
+ throw new ServerError('Transaction was closed while waiting for a record lock', 500);
3156
+ }
3157
+ }
3158
+ link.registerRecordLock(handle);
3159
+ if (link.saveCommits && (context as any)?.timestamp) handle.noteCandidateFloor((context as any).timestamp);
3160
+ if (link.open === TRANSACTION_STATE.OPEN && !link.saveCommits) {
3161
+ // Explicit transaction() (not ImmediateTransaction): pin the clock to
3162
+ // acquiredAt when no writes have been staged yet. When writes already
3163
+ // exist, leave the clock alone (ordering is best-effort; write held records
3164
+ // in their own transaction for the guarantee). ImmediateTransaction is
3165
+ // excluded (saveCommits=true) — its clock is never pinned in lock();
3166
+ // each save() stamps from the handle's committed version floor instead.
3167
+ if (link.writes.length === 0 && !link.timestamp) {
3168
+ link.timestamp = handle.acquiredAt;
3169
+ }
3170
+ if (!resolved.hold && link.transaction) {
3171
+ // Scoped lock: the read snapshot may predate the lock; drop it so the
3172
+ // scope reads what it locked. Hold locks use acquiredAt directly and
3173
+ // do not update the read snapshot.
3174
+ // The timestamp guard matches DatabaseTransaction's own setTimestamp calls: a
3175
+ // deferred update() write leaves the clock at 0, which rocksdb-js rejects.
3176
+ if (link.writes.length === 0 && link.readTxnsUsed <= 1) {
3177
+ link.releaseReadTxn();
3178
+ link.snapshotFree = true;
3179
+ } else if (link.timestamp) link.transaction.setTimestamp(link.timestamp);
3180
+ }
3181
+ }
3182
+ // ImmediateTransaction: no clock pinning in lock(); save() stamps each write
3183
+ // from the committed handle floor for both scoped and hold handles.
3184
+ return handle;
3185
+ });
3186
+ link.registerPendingLock(primaryStore, keyId, acquisition);
3187
+ return acquisition.then(
3188
+ (handle) => {
3189
+ link.unregisterPendingLock(primaryStore, keyId);
3190
+ return this.#reloadLocked(id, handle);
3191
+ },
3192
+ (error) => {
3193
+ link.unregisterPendingLock(primaryStore, keyId);
3194
+ throw error;
3195
+ }
3196
+ );
3197
+ }
3198
+ #reloadLocked(id: Id, holdHandle?: RecordLockHandle | null, preserveChanges = false) {
3199
+ // For freshness, read the committed entry (snapshot-free) so a hold lock sees concurrent
3200
+ // committed writes rather than a stale snapshot. A write earlier in THIS explicit
3201
+ // transaction has not landed in that committed entry yet (harper#1968: Harper defers an
3202
+ // explicit transaction's writes until the writing call actually runs them), so pull the
3203
+ // current value the same way a chained write picks up its basis (priorStagedWrite): the
3204
+ // record comes from the prior staged write, the rest of the entry (version, audit chain,
3205
+ // blob metadata) stays the pre-transaction one.
3206
+ const link = txnForContext(this.getContext());
3207
+ let entryForReload: any = primaryStore.getEntry(id);
3208
+ if (link.open === TRANSACTION_STATE.OPEN) {
3209
+ const keyId = writeKeyId(id);
3210
+ const tailWrite = link.writesByKey?.get(primaryStore)?.get(keyId);
3211
+ const priorStaged =
3212
+ tailWrite && (tailWrite.stagedEntry !== undefined ? tailWrite : priorStagedWrite(tailWrite));
3213
+ if (priorStaged?.stagedEntry !== undefined) {
3214
+ entryForReload = entryForReload
3215
+ ? { ...entryForReload, value: priorStaged.stagedEntry.value }
3216
+ : { value: priorStaged.stagedEntry.value };
3217
+ if (entryForReload.value && typeof entryForReload.value === 'object') {
3218
+ // Register the merged entry in entryMap so getUpdatedTime() works.
3219
+ entryMap.set(entryForReload.value, entryForReload);
3220
+ }
3221
+ }
3222
+ }
3223
+ if (writeKeyId(id) !== writeKeyId(this.getId())) {
3224
+ // lock(target) where target differs from this record: return a separate instance.
3225
+ const fresh = new (this.constructor as any)(id, this.getContext());
3226
+ TableResource._updateResource(fresh, entryForReload);
3227
+ if (holdHandle != null) {
3228
+ fresh.#lockHandle = holdHandle;
3229
+ // Do not clear this.#lockHandle: the original instance keeps its own lock on its
3230
+ // own id; the fresh instance owns the lock on the target id independently.
3231
+ }
3232
+ fresh.#lockWritable = true;
3233
+ // Scoped (not hold) stages exactly like update(): create the TransactionWrite now so
3234
+ // save() is the ordinary #savingOperation path. Hold keeps deferred staging (the
3235
+ // acquiring transaction may commit before the holder ever writes).
3236
+ if (!fresh.#lockHandle!.hold) fresh._writeUpdate(id, fresh.#changes, false);
3237
+ return fresh;
3238
+ }
3239
+ // Store the handle for both scoped and hold locks; undefined (re-entrant hold fast-path)
3240
+ // must not clear a handle already set.
3241
+ if (holdHandle != null) this.#lockHandle = holdHandle;
3242
+ TableResource._updateResource(this, entryForReload);
3243
+ // Preserve staged changes when upgrading the same instance from scoped to hold so that
3244
+ // set() calls made under the scoped lock survive the reload.
3245
+ if (!preserveChanges) this.#changes = undefined;
3246
+ this.#lockWritable = true;
3247
+ // Scoped (not hold): stage now, same as update() would. Skip if a write from an earlier
3248
+ // lock() cycle on this instance is still pending (re-entrant call before its save()).
3249
+ if (!this.#lockHandle!.hold && !this.#savingOperation) this._writeUpdate(id, this.#changes, false);
3250
+ return this;
3251
+ }
3252
+ /**
3253
+ * Release the lock this instance holds. Resolves true when this call cleared the native key lock.
3254
+ * Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
3255
+ * instance is no longer lock-writable; writes through it require a fresh lock.
3256
+ */
3257
+ unlock(): Promise<boolean> {
3258
+ // Always clear the local lock-writable state so subsequent writes on this instance are
3259
+ // ungated, regardless of whether the handle was already released.
3260
+ const handle = this.#lockHandle;
3261
+ this.#lockHandle = undefined;
3262
+ this.#lockWritable = false;
3263
+ if (!handle || handle.released) return Promise.resolve(false);
3264
+ const link = txnForContext(this.getContext());
3265
+ // A scoped lock staged its write at lock() time; released before commit, that write must not
3266
+ // run into the released-handle guard at the sweep.
3267
+ if (this.#savingOperation && !this.#savingOperation.saved && this.#savingOperation.lockHandle === handle)
3268
+ this.#savingOperation = null;
3269
+ detachScopedUpgradeWrite(link, writeKeyId(this.getId()), handle);
3270
+ link.unregisterRecordLock(handle);
3271
+ return Promise.resolve(handle.release());
2208
3272
  }
2209
3273
  static operation(operation, context) {
2210
3274
  operation.table ||= tableName;
@@ -2330,8 +3394,11 @@ export function makeTable(options) {
2330
3394
  // a notification that a write has already occurred in the canonical data source, we need to update our
2331
3395
  // local copy
2332
3396
  _writeUpdate(id: Id, recordUpdate: any, fullUpdate: boolean, options?: any) {
3397
+ this.#assertLiveHandle(id);
2333
3398
  const context = this.getContext();
2334
3399
  const transaction = txnForContext(context);
3400
+ const replaying = transaction.isReplay === true;
3401
+ assertDerivedIndexAdmission(options, transaction);
2335
3402
  checkValidId(id);
2336
3403
  if (fullUpdate && recordUpdate == null && options?.isNotification) {
2337
3404
  // A source/replication-applied put must carry the record; these applies skip record
@@ -2349,6 +3416,16 @@ export function makeTable(options) {
2349
3416
  }
2350
3417
  return;
2351
3418
  }
3419
+ let captureChanges;
3420
+ if (recordUpdate === undefined) {
3421
+ let captured = false;
3422
+ captureChanges = () => {
3423
+ if (!captured) {
3424
+ captured = true;
3425
+ recordUpdate = this.#changes;
3426
+ }
3427
+ };
3428
+ }
2352
3429
  const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
2353
3430
  const writeToSource = () => {
2354
3431
  if (!(this.constructor as any).source || (context as any)?.source) return;
@@ -2368,15 +3445,32 @@ export function makeTable(options) {
2368
3445
  }
2369
3446
  };
2370
3447
 
3448
+ const receiverId = this.getId();
3449
+ const closesReceiver =
3450
+ !this.isCollection &&
3451
+ !isSearchTarget(receiverId) &&
3452
+ (id === receiverId || writeKeyId(id) === writeKeyId(receiverId));
2371
3453
  const write: any = {
2372
3454
  key: id,
2373
3455
  store: primaryStore,
2374
3456
  entry,
2375
3457
  nodeName: (context as any)?.nodeName,
2376
3458
  fullUpdate,
3459
+ chainsStagedState: true,
3460
+ // copy-apply rows keep their pre-read base: one read per row, healed by the post-copy replay
3461
+ reloadCommitBase: options?.isCopyApply !== true,
2377
3462
  deferSave: true,
3463
+ // the origin's record version on an applied write; absent for a locally-originated one
3464
+ recordVersion: options?.version,
3465
+ // Include the lock handle (if any) so the expired-handle guard in
3466
+ // DatabaseTransaction.save() can throw 409 when the lease has lapsed.
3467
+ // Only attach the hold handle when it covers exactly this key; off-key writes
3468
+ // are ordinary and must not carry an unrelated hold's handle.
3469
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
3470
+ writeGeneration: !this.#lockWritable && closesReceiver ? this[GET_TRACKED_WRITE_GENERATION]() : undefined,
3471
+ captureChanges,
2378
3472
  validate: (txnTime, committedBy = transaction) => {
2379
- if (!recordUpdate) recordUpdate = this.#changes;
3473
+ write.captureChanges?.();
2380
3474
  if (fullUpdate || (recordUpdate && hasChanges(this.#changes === recordUpdate ? this : recordUpdate))) {
2381
3475
  if (!(context as any)?.source) {
2382
3476
  committedBy.checkOverloaded();
@@ -2427,10 +3521,13 @@ export function makeTable(options) {
2427
3521
  : txnTime;
2428
3522
  }
2429
3523
  if (createdTimeProperty) {
2430
- if (entry?.value) {
3524
+ // the reloaded commit base, not the pre-read one: a full PUT racing a create
3525
+ // would otherwise stamp a fresh created time over the real one
3526
+ const base = write.entry;
3527
+ if (base?.value) {
2431
3528
  if (fullUpdate || recordUpdate[createdTimeProperty.name]) {
2432
3529
  // make sure to retain original created time
2433
- recordUpdate[createdTimeProperty.name] = entry?.value[createdTimeProperty.name];
3530
+ recordUpdate[createdTimeProperty.name] = base.value[createdTimeProperty.name];
2434
3531
  }
2435
3532
  } else {
2436
3533
  // new entry, set created time
@@ -2497,6 +3594,8 @@ export function makeTable(options) {
2497
3594
  this.#savingOperation = null;
2498
3595
  write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
2499
3596
  let omitLocalRecord = false;
3597
+ const txnLogKey =
3598
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2500
3599
  // we use optimistic locking to only commit if the existing record state still holds true.
2501
3600
  // this is superior to using an async transaction since it doesn't require JS execution
2502
3601
  // during the write transaction.
@@ -2559,10 +3658,10 @@ export function makeTable(options) {
2559
3658
  if (
2560
3659
  existingEntry.additionalAuditRefs?.some(
2561
3660
  (ref) =>
2562
- ref.version === txnTime &&
3661
+ ref.version === txnLogKey &&
2563
3662
  precedesExistingVersion(
2564
3663
  txnTime,
2565
- { version: txnTime, localTime: txnTime, key: id, nodeId: ref.nodeId },
3664
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId },
2566
3665
  options?.nodeId
2567
3666
  ) === 0
2568
3667
  )
@@ -2593,10 +3692,10 @@ export function makeTable(options) {
2593
3692
  if (!oldestRetainedAuditTimeResolved) {
2594
3693
  oldestRetainedAuditTimeResolved = true;
2595
3694
  // getRange yields ascending by audit-log key, so the first entry is the oldest retained.
2596
- // Mirror replicationConnection's retention check and the cleanup key basis (localTime ??
2597
- // version). Fall back to the nominal time-based purge floor when the log is empty/unavailable.
3695
+ // Mirror replicationConnection's retention check and the cleanup key basis (`txnLogKey`).
3696
+ // Fall back to the nominal time-based purge floor when the log is empty/unavailable.
2598
3697
  for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
2599
- oldestRetainedAuditTime = entry.localTime ?? entry.version;
3698
+ oldestRetainedAuditTime = entry.txnLogKey;
2600
3699
  break;
2601
3700
  }
2602
3701
  oldestRetainedAuditTime ??= Date.now() - auditRetention;
@@ -2609,23 +3708,25 @@ export function makeTable(options) {
2609
3708
  // depth-cap block. This is the same keyed lookup that block performs, hoisted ahead of the walk.
2610
3709
  // It is what catches transitive/proxied re-deliveries: they arrive buried below the record head
2611
3710
  // (so replication's head-tie fast-skip can't see them) yet are exact duplicates. Keyed by nodeId,
2612
- // so it is correct across multiple source nodes. RocksDB-only: LMDB audit entries are keyed by
2613
- // local audit time, not version, so this version-keyed lookup doesn't apply there (LMDB keeps the
2614
- // exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
3711
+ // so it is correct across multiple source nodes. The lookup key is this write's LOG key, not its
3712
+ // record version — a replication apply commits under the origin's log key while storing the
3713
+ // origin's version, and only the log key addresses the entry (harper#2412).
3714
+ // RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
3715
+ // there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
2615
3716
  // simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
2616
3717
  // check above remains the read-your-writes guard. Never when this write staged in a prior
2617
3718
  // failed attempt: that attempt already appended this write's own audit entry, so the lookup
2618
3719
  // would find it and skip the write as "already applied" when the record was never committed.
2619
3720
  // A recommit of the same transaction survived that skip only because the old write batch
2620
3721
  // still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
2621
- if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnTime)) {
2622
- const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId);
3722
+ if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
3723
+ const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
2623
3724
  if (
2624
3725
  priorAudit &&
2625
- priorAudit.version === txnTime &&
3726
+ priorAudit.txnLogKey === txnLogKey &&
2626
3727
  precedesExistingVersion(
2627
3728
  txnTime,
2628
- { version: txnTime, localTime: txnTime, key: id, nodeId: priorAudit.nodeId },
3729
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: priorAudit.nodeId },
2629
3730
  options?.nodeId
2630
3731
  ) === 0
2631
3732
  ) {
@@ -2634,7 +3735,10 @@ export function makeTable(options) {
2634
3735
  }
2635
3736
  }
2636
3737
  // incremental CRDT updates are only available with audit logging on
2637
- let localTime = existingEntry.localTime;
3738
+ const initialAuditHead = isRocksDB
3739
+ ? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
3740
+ : { txnLogKey: existingEntry.localTime, nodeId: existingEntry.nodeId };
3741
+ let localTime = initialAuditHead.txnLogKey;
2638
3742
  let auditedVersion = existingEntry.version;
2639
3743
  logger.debug?.(
2640
3744
  'Applying CRDT update to record with id: ',
@@ -2647,22 +3751,42 @@ export function makeTable(options) {
2647
3751
  new Date(localTime)
2648
3752
  );
2649
3753
 
2650
- let nodeId = existingEntry.nodeId;
3754
+ let nodeId = initialAuditHead.nodeId;
2651
3755
  const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
2652
3756
  const auditRefsToVisit: Array<{ localTime: number; nodeId: number }> = existingEntry.additionalAuditRefs
2653
3757
  ? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
2654
3758
  : [];
2655
3759
 
2656
- // Collect any existing audit refs that should be preserved (those older than current transaction)
3760
+ // Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
2657
3761
  if (existingEntry.additionalAuditRefs) {
2658
3762
  for (const ref of existingEntry.additionalAuditRefs) {
2659
- if (ref.version <= txnTime) {
2660
- additionalAuditRefs.push(ref);
2661
- }
3763
+ additionalAuditRefs.push(ref);
2662
3764
  }
2663
3765
  }
2664
3766
  let addedAuditRef = false;
2665
3767
  let nextRef: { localTime: number; nodeId: number };
3768
+ const visitedAuditRefs = new Set<string>();
3769
+ const queuePreviousAuditRefs = (auditRecord) => {
3770
+ const previousRefs = auditRecord.previousAdditionalAuditRefs;
3771
+ if (previousRefs) {
3772
+ for (const ref of previousRefs) {
3773
+ auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
3774
+ logger.debug?.('Adding audit ref from audit record to visit queue', {
3775
+ version: ref.version,
3776
+ nodeId: ref.nodeId,
3777
+ });
3778
+ }
3779
+ }
3780
+ };
3781
+ const advanceToPreviousAudit = (auditRecord) => {
3782
+ const previousRefs = auditRecord.previousAdditionalAuditRefs;
3783
+ const previousHead =
3784
+ isRocksDB && previousRefs?.length
3785
+ ? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, previousRefs)
3786
+ : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
3787
+ localTime = previousHead.txnLogKey;
3788
+ nodeId = previousHead.nodeId;
3789
+ };
2666
3790
  let walkSteps = 0;
2667
3791
  let auditWalkCapped = false;
2668
3792
  // Early-out residual: as we walk the chain newest-first, fold each succeeding patch into a
@@ -2681,21 +3805,24 @@ export function makeTable(options) {
2681
3805
  // appended this write's own audit entry, so the lookup would match it while the record was
2682
3806
  // never committed (see the up-front keyed dedup above).
2683
3807
  const isReDeliveredDuplicate = () => {
2684
- if (stagedOwnAuditEntry) return false;
2685
- if (!dedupVersionCouldBeRetained(txnTime)) return false; // pre-retention version — skip the end-of-log scan (best-effort; see above)
2686
- const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId);
3808
+ if (replaying || stagedOwnAuditEntry) return false;
3809
+ if (!dedupVersionCouldBeRetained(txnLogKey)) return false; // pre-retention log key — skip the end-of-log scan (best-effort; see above)
3810
+ const duplicate = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
2687
3811
  return (
2688
3812
  duplicate &&
2689
- duplicate.version === txnTime &&
3813
+ duplicate.txnLogKey === txnLogKey &&
2690
3814
  precedesExistingVersion(
2691
3815
  txnTime,
2692
- { version: txnTime, localTime: txnTime, key: id, nodeId: duplicate.nodeId },
3816
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: duplicate.nodeId },
2693
3817
  options?.nodeId
2694
3818
  ) === 0
2695
3819
  );
2696
3820
  };
2697
3821
  do {
2698
3822
  while (localTime > txnTime || (auditedVersion >= txnTime && localTime > 0)) {
3823
+ const auditIdentity = `${nodeId ?? 0}:${localTime}`;
3824
+ if (visitedAuditRefs.has(auditIdentity)) break;
3825
+ visitedAuditRefs.add(auditIdentity);
2699
3826
  // Bound the walk only for RocksDB, where the OOM was observed (issue #1114): each step
2700
3827
  // is a transaction-log range scan + msgpackr decode, and the per-node logs can be huge.
2701
3828
  // LMDB audit entries are keyed by local audit time (not version), so the duplicate
@@ -2706,6 +3833,21 @@ export function makeTable(options) {
2706
3833
  }
2707
3834
  const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
2708
3835
  if (!auditRecord) break;
3836
+ queuePreviousAuditRefs(auditRecord);
3837
+ if (
3838
+ isRocksDB &&
3839
+ !replaying &&
3840
+ !stagedOwnAuditEntry &&
3841
+ localTime === txnLogKey &&
3842
+ precedesExistingVersion(
3843
+ txnTime,
3844
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId },
3845
+ options?.nodeId
3846
+ ) === 0
3847
+ ) {
3848
+ write.skipped = true;
3849
+ return;
3850
+ }
2709
3851
  auditedVersion = auditRecord.version;
2710
3852
  if (auditedVersion >= txnTime) {
2711
3853
  if (auditedVersion === txnTime) {
@@ -2715,17 +3857,26 @@ export function makeTable(options) {
2715
3857
  options?.nodeId
2716
3858
  );
2717
3859
  if (precedesExisting === 0) {
2718
- logger.debug?.(
2719
- 'The transaction time is equal to the existing version, treating as duplicate',
2720
- id
2721
- );
2722
- write.skipped = true;
2723
- return; // treat a tie as a duplicate and drop it
3860
+ if (isRocksDB && localTime !== txnLogKey) {
3861
+ // Same origin and record version, but a distinct write. Its per-origin log key
3862
+ // orders the otherwise non-unique record clock without comparing keys across origins.
3863
+ precedesExisting = txnLogKey > localTime ? 1 : -1;
3864
+ } else if (replaying || stagedOwnAuditEntry) {
3865
+ // The log entry being replayed (or staged by this write's failed attempt) is
3866
+ // the write itself, not proof that its primary-store mutation committed.
3867
+ precedesExisting = 1;
3868
+ } else {
3869
+ logger.debug?.(
3870
+ 'The transaction time and log key match the existing write, treating as duplicate',
3871
+ id
3872
+ );
3873
+ write.skipped = true;
3874
+ return;
3875
+ }
2724
3876
  }
2725
3877
  if (precedesExisting > 0) {
2726
3878
  // if the existing version is older, we can skip this update
2727
- localTime = auditRecord.previousVersion;
2728
- nodeId = auditRecord.previousNodeId;
3879
+ advanceToPreviousAudit(auditRecord);
2729
3880
  continue;
2730
3881
  }
2731
3882
  }
@@ -2762,24 +3913,16 @@ export function makeTable(options) {
2762
3913
  }
2763
3914
  if (!addedAuditRef && isRocksDB) {
2764
3915
  addedAuditRef = true;
2765
- // Add a reference to this older audit record if we had out-of-order writes
2766
- additionalAuditRefs.push({ version: txnTime, nodeId: options?.nodeId });
3916
+ // Add a reference to this older audit record if we had out-of-order writes. The stored
3917
+ // value is a LOG key, not a record version: every consumer follows it straight into
3918
+ // `auditStore.get` (see the `auditRefsToVisit` mapping above and below), and on an
3919
+ // applied write those two clocks differ.
3920
+ additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
2767
3921
  logger.debug?.('Adding additional audit ref for out-of-order write', {
2768
- version: txnTime,
3922
+ txnLogKey,
2769
3923
  nodeId: options?.nodeId,
2770
3924
  });
2771
3925
  }
2772
- // Collect any additional audit refs from this audit record to traverse other branches
2773
- if (auditRecord.previousAdditionalAuditRefs) {
2774
- for (const ref of auditRecord.previousAdditionalAuditRefs) {
2775
- auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
2776
- logger.debug?.('Adding audit ref from audit record to visit queue', {
2777
- version: ref.version,
2778
- nodeId: ref.nodeId,
2779
- });
2780
- }
2781
- }
2782
-
2783
3926
  // Every field of this write is overwritten by newer writes, and there is no alternate
2784
3927
  // audit branch left to scan, so it is fully superseded — the same outcome as walking to
2785
3928
  // the end and taking the `writeCommit(false)` escape below, reached without paying the rest
@@ -2798,8 +3941,7 @@ export function makeTable(options) {
2798
3941
  return writeCommit(false);
2799
3942
  }
2800
3943
 
2801
- localTime = auditRecord.previousVersion;
2802
- nodeId = auditRecord.previousNodeId;
3944
+ advanceToPreviousAudit(auditRecord);
2803
3945
  }
2804
3946
  // Check if we need to scan additional audit refs from this record
2805
3947
  if (auditWalkCapped) break;
@@ -2904,8 +4046,8 @@ export function makeTable(options) {
2904
4046
  if (recordToStore && recordToStore.getRecord)
2905
4047
  throw new Error('Can not assign a record to a record, check for circular references');
2906
4048
  if (residencyId == undefined) {
2907
- if (entry?.residencyId)
2908
- (context as any).previousResidency = TableResource.getResidencyRecord(entry.residencyId);
4049
+ if (existingEntry?.residencyId)
4050
+ (context as any).previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
2909
4051
  const residency = residencyFromFunction(TableResource.getResidency(recordToStore, context));
2910
4052
  if (residency) {
2911
4053
  if (!residency.includes(server.hostname)) {
@@ -2986,7 +4128,16 @@ export function makeTable(options) {
2986
4128
  );
2987
4129
  updateIndices(id, existingRecord, recordToStore, transaction && { transaction });
2988
4130
 
4131
+ // Preserve an addressable audit head when the record and log clocks diverge.
4132
+ if (isRocksDB && audit && !isCopyApply && txnLogKey !== txnTime) {
4133
+ const headIndex = additionalAuditRefs.findIndex(
4134
+ (ref) => ref.version === txnLogKey && (ref.nodeId ?? 0) === (options?.nodeId ?? 0)
4135
+ );
4136
+ if (headIndex > 0) additionalAuditRefs.unshift(additionalAuditRefs.splice(headIndex, 1)[0]);
4137
+ else if (headIndex < 0) additionalAuditRefs.unshift({ version: txnLogKey, nodeId: options?.nodeId });
4138
+ }
2989
4139
  writeCommit(true);
4140
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
2990
4141
  if (expiresAt >= 0) {
2991
4142
  scheduleCleanup(); // arm for replicated writes too, not just local-context writes
2992
4143
  // A runtime per-record expiresAt on a table with no table-level expiration/eviction, no expiresAt
@@ -3021,6 +4172,8 @@ export function makeTable(options) {
3021
4172
  user: (context as any)?.user,
3022
4173
  residencyId,
3023
4174
  expiresAt,
4175
+ recordVersion: txnTime,
4176
+ recordNodeId: precedesExisting < 0 ? existingEntry?.nodeId : options?.nodeId,
3024
4177
  nodeId: options?.nodeId,
3025
4178
  viaNodeId: options?.viaNodeId,
3026
4179
  originatingOperation: (context as any)?.originatingOperation,
@@ -3131,8 +4284,10 @@ export function makeTable(options) {
3131
4284
  return Boolean(this.#record);
3132
4285
  }
3133
4286
  _writeDelete(id: Id, options?: any) {
4287
+ this.#assertLiveHandle(id);
3134
4288
  const context = this.getContext();
3135
4289
  const transaction = txnForContext(context);
4290
+ assertDerivedIndexAdmission(options, transaction);
3136
4291
  checkValidId(id);
3137
4292
  const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
3138
4293
 
@@ -3141,7 +4296,10 @@ export function makeTable(options) {
3141
4296
  store: primaryStore,
3142
4297
  entry,
3143
4298
  chainsStagedState: true,
4299
+ reloadCommitBase: true,
3144
4300
  nodeName: (context as any)?.nodeName,
4301
+ recordVersion: options?.version,
4302
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
3145
4303
  before:
3146
4304
  (this.constructor as any).source?.delete && !(context as any)?.source
3147
4305
  ? (this.constructor as any).source.delete.bind((this.constructor as any).source, id, undefined, context)
@@ -3154,6 +4312,8 @@ export function makeTable(options) {
3154
4312
  const priorStagedOp = priorStagedWrite(write);
3155
4313
  const priorStaged = priorStagedOp?.stagedEntry;
3156
4314
  const existingRecord = priorStaged ? priorStaged.value : existingEntry?.value;
4315
+ const txnLogKey =
4316
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
3157
4317
  if (retry) {
3158
4318
  if (context && existingEntry?.version > (context.lastModified || 0))
3159
4319
  context.lastModified = existingEntry.version;
@@ -3182,6 +4342,11 @@ export function makeTable(options) {
3182
4342
  viaNodeId: options?.viaNodeId,
3183
4343
  transaction,
3184
4344
  tableToTrack: tableName,
4345
+ recordVersion: txnTime,
4346
+ additionalAuditRefs:
4347
+ isRocksDB && audit && txnLogKey !== txnTime
4348
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
4349
+ : undefined,
3185
4350
  },
3186
4351
  'delete'
3187
4352
  );
@@ -3191,6 +4356,7 @@ export function makeTable(options) {
3191
4356
  removeEntry(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
3192
4357
  }
3193
4358
  write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
4359
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
3194
4360
  // the removal supersedes the nearest record an earlier write in this transaction stored
3195
4361
  // (older ones were already marked by their staged successors), so its saved blobs are
3196
4362
  // cleaned up post-commit unless its audit entry references them
@@ -3504,7 +4670,13 @@ export function makeTable(options) {
3504
4670
  404
3505
4671
  );
3506
4672
  }
3507
- if (orderAlignedCondition) orderAlignedCondition.descending = Boolean(sort.descending);
4673
+ if (orderAlignedCondition) {
4674
+ orderAlignedCondition.descending = Boolean(sort.descending);
4675
+ if (orderAlignedCondition.maxIndexLagMilliseconds === undefined)
4676
+ orderAlignedCondition.maxIndexLagMilliseconds = sort.maxIndexLagMilliseconds;
4677
+ if (orderAlignedCondition.waitForIndexMilliseconds === undefined)
4678
+ orderAlignedCondition.waitForIndexMilliseconds = sort.waitForIndexMilliseconds;
4679
+ }
3508
4680
  }
3509
4681
  }
3510
4682
  conditions = orderConditions(conditions, operator);
@@ -3531,6 +4703,10 @@ export function makeTable(options) {
3531
4703
  }
3532
4704
  }
3533
4705
  const select = target.select;
4706
+ // Whether the caller supplied real filter conditions — read from the raw request, NOT the
4707
+ // planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
4708
+ // injected full-scan condition). Used to pick the count-estimate source below.
4709
+ const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
3534
4710
  if (conditions.length === 0) {
3535
4711
  conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
3536
4712
  }
@@ -3568,65 +4744,175 @@ export function makeTable(options) {
3568
4744
  boundRowFilter || typeof target.vectorFilter === 'function'
3569
4745
  ? { rowFilter: boundRowFilter, vectorFilter: target.vectorFilter }
3570
4746
  : undefined;
3571
- const entries = executeConditions(
3572
- conditions,
3573
- operator,
3574
- TableResource,
3575
- readTxn,
3576
- target,
3577
- context,
3578
- (results: any[], filters: Function[]) => transformToEntries(results, select, context, readTxn, filters),
3579
- filtered,
3580
- recordAccess
3581
- );
3582
- const ensure_loaded = (target as any).ensureLoaded !== false;
3583
- // The guards inside executeConditions evaluate the
3584
- // LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
3585
- // expired/invalidated row from source and return a DIFFERENT record. The explicit row filter
3586
- // must hold on the record actually returned, so it is re-checked
3587
- // there, after materialization (the earlier evaluation stays as a prune that also bounds HNSW
3588
- // traversal). vectorFilter and condition filters intentionally keep the local-record
3589
- // semantics all query filters have on caching tables.
3590
- //
3591
- // A row that is past its TTL but not yet swept by the background eviction
3592
- // scan is still physically present. A write that is about to overwrite it
3593
- // anyway (e.g. the SQL engine locating UPDATE/DELETE targets) needs to see
3594
- // it as a match — the same leniency a direct by-id put/patch already gets,
3595
- // since those never run the ensureLoaded-gated freshness check this transform
3596
- // otherwise applies unconditionally to every read.
3597
- const includeExpired = (target as any).includeExpired === true;
3598
- const transformToRecord = TableResource.transformEntryForSelect(
3599
- select,
3600
- context,
3601
- readTxn,
3602
- filtered,
3603
- ensure_loaded,
3604
- true,
3605
- boundRowFilter,
3606
- includeExpired,
3607
- postOrdering
3608
- );
3609
- let results = TableResource.transformToOrderedSelect(
3610
- entries,
3611
- select,
3612
- postOrdering,
3613
- context,
3614
- readTxn,
3615
- transformToRecord
3616
- );
3617
- // apply any offset/limit after all the sorting and filtering
3618
- if (target.offset || target.limit !== undefined)
3619
- results = results.slice(
3620
- target.offset,
3621
- target.limit !== undefined ? (target.offset || 0) + target.limit : undefined
4747
+ try {
4748
+ const entries = executeConditions(
4749
+ conditions,
4750
+ operator,
4751
+ TableResource,
4752
+ readTxn,
4753
+ target,
4754
+ context,
4755
+ (results: any[], filters: Function[]) => transformToEntries(results, select, context, readTxn, filters),
4756
+ filtered,
4757
+ recordAccess
4758
+ );
4759
+ const ensure_loaded = (target as any).ensureLoaded !== false;
4760
+ // The guards inside executeConditions evaluate the
4761
+ // LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
4762
+ // expired/invalidated row from source and return a DIFFERENT record. The explicit row filter
4763
+ // must hold on the record actually returned, so it is re-checked
4764
+ // there, after materialization (the earlier evaluation stays as a prune that also bounds HNSW
4765
+ // traversal). vectorFilter and condition filters intentionally keep the local-record
4766
+ // semantics all query filters have on caching tables.
4767
+ //
4768
+ // A row that is past its TTL but not yet swept by the background eviction
4769
+ // scan is still physically present. A write that is about to overwrite it
4770
+ // anyway (e.g. the SQL engine locating UPDATE/DELETE targets) needs to see
4771
+ // it as a match — the same leniency a direct by-id put/patch already gets,
4772
+ // since those never run the ensureLoaded-gated freshness check this transform
4773
+ // otherwise applies unconditionally to every read.
4774
+ const includeExpired = (target as any).includeExpired === true;
4775
+ const transformToRecord = TableResource.transformEntryForSelect(
4776
+ select,
4777
+ context,
4778
+ readTxn,
4779
+ filtered,
4780
+ ensure_loaded,
4781
+ true,
4782
+ boundRowFilter,
4783
+ includeExpired,
4784
+ postOrdering
4785
+ );
4786
+ let results = TableResource.transformToOrderedSelect(
4787
+ entries,
4788
+ select,
4789
+ postOrdering,
4790
+ context,
4791
+ readTxn,
4792
+ transformToRecord
3622
4793
  );
3623
- results.onDone = () => {
3624
- results.onDone = null; // ensure that it isn't called twice
4794
+ const offset = target.offset || 0;
4795
+ const end = target.limit !== undefined ? offset + (target.limit as number) : undefined;
4796
+ // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
4797
+ // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
4798
+ // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
4799
+ // table estimate. Opt-in only — the default streaming path below is untouched.
4800
+ //
4801
+ // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
4802
+ // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
4803
+ // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
4804
+ // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
4805
+ // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
4806
+ // normal streaming path with no count. This bounds the offset too: without it a huge offset would
4807
+ // postpone the exact guardrail (which only engages past the page) until that offset was scanned.
4808
+ const pageLimit = target.limit as number;
4809
+ if (
4810
+ target.count &&
4811
+ Number.isInteger(pageLimit) &&
4812
+ pageLimit >= 0 &&
4813
+ pageLimit <= MAX_COUNT_PAGE &&
4814
+ Number.isInteger(offset) &&
4815
+ offset >= 0 &&
4816
+ offset + pageLimit <= MAX_EXACT_COUNT_SCAN
4817
+ ) {
4818
+ const wantExact = target.count === 'exact';
4819
+ const pageEnd = offset + pageLimit;
4820
+ const countStart = performance.now();
4821
+ // A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
4822
+ // chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
4823
+ // the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
4824
+ // different `count=exact` totals. Any query whose execution touches a custom index is affected: a
4825
+ // custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
4826
+ // filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
4827
+ // filter. Report the total as unavailable for those rather than advertising it as count=exact
4828
+ // (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
4829
+ // sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
4830
+ const touchesCustomIndex = (conds: any[]): boolean =>
4831
+ conds.some((c: any) => {
4832
+ if (!c) return false;
4833
+ if (c.conditions) return touchesCustomIndex(c.conditions);
4834
+ const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
4835
+ return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
4836
+ });
4837
+ const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
4838
+ return (async () => {
4839
+ const page: any = [];
4840
+ let scanned = 0;
4841
+ let exact = true;
4842
+ try {
4843
+ for await (const record of results) {
4844
+ if (scanned >= offset && scanned < pageEnd) page.push(record);
4845
+ scanned++;
4846
+ // A store whose async iterator settles synchronously (the common indexed-scan case) would
4847
+ // otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
4848
+ // for the whole count. Yield to the macrotask queue periodically so concurrent requests and
4849
+ // I/O still make progress during a large exact scan.
4850
+ if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0) await new Promise((resolve) => setImmediate(resolve));
4851
+ // The page window [offset, pageEnd) is always collected in full first — the guardrail
4852
+ // only ever abandons the running TOTAL, never truncates the page body.
4853
+ if (scanned >= pageEnd) {
4854
+ // `estimated` needs nothing past the page; an approximate (vector) exact total is going to
4855
+ // be reported unavailable anyway, so don't drain its tail for a number we won't publish.
4856
+ if (!wantExact || approximateResultSet) break;
4857
+ // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
4858
+ // large match set can't turn a bounded page fetch into an unbounded scan.
4859
+ if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) {
4860
+ exact = false;
4861
+ break;
4862
+ }
4863
+ }
4864
+ }
4865
+ } finally {
4866
+ // We own the iteration here (no results.onDone consumer), so release the read
4867
+ // transaction unconditionally — including when the drain throws — or the snapshot leaks.
4868
+ txn.doneReadTxn();
4869
+ }
4870
+ let total: number | null;
4871
+ if (wantExact) {
4872
+ // `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
4873
+ // an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
4874
+ total = exact && !approximateResultSet ? scanned : null;
4875
+ } else if (boundRowFilter || typeof target.vectorFilter === 'function') {
4876
+ // An opaque row/vector filter shapes the result but isn't reflected in the index/condition
4877
+ // estimate; guessing would both mislead and disclose cardinality the filter hides.
4878
+ total = null;
4879
+ } else if (!hasUserConditions) {
4880
+ total = estimatedEntryCount(primaryStore);
4881
+ } else {
4882
+ // Estimate from the real conditions only — drop the planner's synthetic `sort`
4883
+ // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
4884
+ const est = estimateCondition(TableResource)({
4885
+ conditions: conditions.filter((c: any) => c.comparator !== 'sort'),
4886
+ operator: operator ? String(operator).toLowerCase() : 'and',
4887
+ });
4888
+ total = isFinite(est) ? Math.round(est) : null;
4889
+ }
4890
+ // For an estimate, never report a total below the last row actually returned — keeps the
4891
+ // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
4892
+ // Exact totals are authoritative (and an empty page past the end must not be clamped up).
4893
+ if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
4894
+ total = offset + page.length;
4895
+ }
4896
+ page.recordCount = total;
4897
+ page.recordCountExact = wantExact && exact && !approximateResultSet;
4898
+ page.selectApplied = true;
4899
+ page.getColumns = getColumns;
4900
+ return page;
4901
+ })() as any;
4902
+ }
4903
+ // apply any offset/limit after all the sorting and filtering
4904
+ if (target.offset || target.limit !== undefined) results = results.slice(offset, end);
4905
+ results.onDone = () => {
4906
+ results.onDone = null; // ensure that it isn't called twice
4907
+ txn.doneReadTxn();
4908
+ };
4909
+ results.selectApplied = true;
4910
+ results.getColumns = getColumns;
4911
+ return results;
4912
+ } catch (error) {
3625
4913
  txn.doneReadTxn();
3626
- };
3627
- results.selectApplied = true;
3628
- results.getColumns = getColumns;
3629
- return results;
4914
+ throw error;
4915
+ }
3630
4916
  }
3631
4917
  /**
3632
4918
  * This is responsible for ordering and select()ing the attributes/properties from returned entries
@@ -3649,10 +4935,17 @@ export function makeTable(options) {
3649
4935
  if (sort) {
3650
4936
  // there might be some situations where we don't need to transform to entries for sorting, not sure
3651
4937
  entries = transformToEntries(entries, select, context, readTxn, null);
3652
- let ordered;
4938
+ // Sort keys are resolved as entries are collected, so comparison never dereferences a record: a
4939
+ // cached entry holds its record only weakly, and a re-read per comparison is what this avoids.
4940
+ const clauses: Sort[] = [];
4941
+ for (let order = sort; order; order = order.next) clauses.push(order);
4942
+ const clauseCount = clauses.length;
3653
4943
  // if we are doing post-ordering, we need to get records first, then sort them
3654
4944
  results.iterate = function (options: { async: boolean }) {
3655
- let sortedArrayIterator: IterableIterator<any>;
4945
+ let ordered: any[];
4946
+ let orderedKeys: any[][];
4947
+ let sortedPositions: number[];
4948
+ let sortedIndex: number;
3656
4949
  const dbIterator =
3657
4950
  options?.async && entries[Symbol.asyncIterator]
3658
4951
  ? entries[Symbol.asyncIterator]()
@@ -3662,25 +4955,33 @@ export function makeTable(options) {
3662
4955
  let enqueuedEntryForNextGroup: any;
3663
4956
  let lastGroupingValue: any;
3664
4957
  let firstEntry = true;
3665
- function createComparator(order: Sort) {
3666
- const nextComparator = order.next && createComparator(order.next);
3667
- const descending = order.descending;
3668
- return (entryA, entryB) => {
3669
- const a = getAttributeValue(entryA, order.attribute, context, order);
3670
- const b = getAttributeValue(entryB, order.attribute, context, order);
3671
- const diff = descending
3672
- ? compareKeys(convertToComparableKeys(b), convertToComparableKeys(a))
3673
- : compareKeys(convertToComparableKeys(a), convertToComparableKeys(b));
3674
- if (diff === 0) return nextComparator?.(entryA, entryB) || 0;
3675
- return diff;
3676
- };
4958
+ function collect(entry) {
4959
+ ordered.push(entry);
4960
+ for (let i = 0; i < clauseCount; i++) {
4961
+ const clause = clauses[i];
4962
+ orderedKeys[i].push(convertToComparableKeys(getAttributeValue(entry, clause.attribute, context, clause)));
4963
+ }
4964
+ }
4965
+ function comparePositions(positionA: number, positionB: number): number {
4966
+ for (let i = 0; i < clauseCount; i++) {
4967
+ const keys = orderedKeys[i];
4968
+ const diff = clauses[i].descending
4969
+ ? compareKeys(keys[positionB], keys[positionA])
4970
+ : compareKeys(keys[positionA], keys[positionB]);
4971
+ if (diff !== 0) return diff;
4972
+ }
4973
+ return 0;
4974
+ }
4975
+ function nextSorted(): IteratorResult<any> {
4976
+ if (sortedIndex < sortedPositions.length)
4977
+ return { done: false, value: ordered[sortedPositions[sortedIndex++]] };
4978
+ return { done: true, value: undefined };
3677
4979
  }
3678
- const comparator = createComparator(sort);
3679
4980
  return {
3680
4981
  async next() {
3681
4982
  let iteration: IteratorResult<any>;
3682
- if (sortedArrayIterator) {
3683
- iteration = sortedArrayIterator.next();
4983
+ if (sortedPositions) {
4984
+ iteration = nextSorted();
3684
4985
  if (iteration.done) {
3685
4986
  if (dbDone) {
3686
4987
  if (results.onDone) results.onDone();
@@ -3692,7 +4993,9 @@ export function makeTable(options) {
3692
4993
  };
3693
4994
  }
3694
4995
  ordered = [];
3695
- if (enqueuedEntryForNextGroup) ordered.push(enqueuedEntryForNextGroup);
4996
+ orderedKeys = [];
4997
+ for (let i = 0; i < clauseCount; i++) orderedKeys.push([]);
4998
+ if (enqueuedEntryForNextGroup) collect(enqueuedEntryForNextGroup);
3696
4999
  // need to load all the entries into ordered
3697
5000
  do {
3698
5001
  iteration = await dbIterator.next();
@@ -3723,17 +5026,17 @@ export function makeTable(options) {
3723
5026
  break;
3724
5027
  }
3725
5028
  }
3726
- // we store the value we will sort on, for fast sorting, and the entry so the records can be GC'ed if necessary
3727
- // before the sorting is completed
3728
- ordered.push(entry);
5029
+ collect(entry);
3729
5030
  }
3730
5031
  } while (true);
3731
5032
  if ((sort as any).isGrouped) {
3732
5033
  // TODO: Return grouped results
3733
5034
  }
3734
- ordered.sort(comparator);
3735
- sortedArrayIterator = ordered[Symbol.iterator]();
3736
- iteration = sortedArrayIterator.next();
5035
+ sortedPositions = [];
5036
+ for (let i = 0; i < ordered.length; i++) sortedPositions.push(i);
5037
+ sortedPositions.sort(comparePositions);
5038
+ sortedIndex = 0;
5039
+ iteration = nextSorted();
3737
5040
  if (!iteration.done)
3738
5041
  return {
3739
5042
  value: await transformToRecord.call(this, iteration.value),
@@ -4078,6 +5381,10 @@ export function makeTable(options) {
4078
5381
  }
4079
5382
  if (!auditStore) throw new Error('Can not subscribe to a table without an audit log');
4080
5383
  if (!audit) {
5384
+ // Turning auditing on is a schema write, and a branch's Table classes carry the base's
5385
+ // logical name: without this a subscribe through a branched application would enable
5386
+ // auditing on the live base table for every other consumer, with no DDL call involved.
5387
+ TableResource.assertSchemaMutable('enable auditing for a subscription');
4081
5388
  table({ table: tableName, database: databaseName, schemaDefined, attributes, audit: true });
4082
5389
  }
4083
5390
  const getFullRecord = !request.rawEvents;
@@ -4132,10 +5439,12 @@ export function makeTable(options) {
4132
5439
  const subscription = addSubscription(
4133
5440
  TableResource,
4134
5441
  thisId,
4135
- function (id: Id, auditRecord?: any, localTime?: any, beginTxn?: any) {
5442
+ function (id: Id, auditRecord?: any, txnLogKey?: any, beginTxn?: any) {
4136
5443
  if (dropDuringReplay) return;
4137
5444
  try {
4138
5445
  let type = auditRecord.type;
5446
+ // Ahead of the rawEvents branch, which forwards every type verbatim.
5447
+ if (isLockControlType(type)) return;
4139
5448
  let value;
4140
5449
  if (type === 'message' || request.rawEvents) {
4141
5450
  // we only send the full message, this are individual messages that can be sent out of order
@@ -4155,8 +5464,7 @@ export function makeTable(options) {
4155
5464
  // been written, so are fresh in memory.
4156
5465
  const entry: Entry = primaryStore.getEntry(id);
4157
5466
  if (entry) {
4158
- // staleness is a record-version comparison; auditRecord.version is the log key on RocksDB
4159
- if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version)) return; // out of order event, with old update, don't send anything
5467
+ if (entry.version !== auditRecord.version) return; // out of order event, with old update, don't send anything
4160
5468
  value = entry.value;
4161
5469
  type = entry.metadataFlags & INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
4162
5470
  } else {
@@ -4165,7 +5473,7 @@ export function makeTable(options) {
4165
5473
  }
4166
5474
  const event = {
4167
5475
  id,
4168
- localTime,
5476
+ localTime: txnLogKey,
4169
5477
  value,
4170
5478
  version: auditRecord.version,
4171
5479
  type,
@@ -4227,14 +5535,15 @@ export function makeTable(options) {
4227
5535
  await rest();
4228
5536
  if (!isActive()) return;
4229
5537
  }
4230
- if (auditRecord.tableId !== tableId) continue;
5538
+ if (auditRecord.tableId !== tableId || auditRecord.type === 'evict') continue;
5539
+ if (isLockControlType(auditRecord.type)) continue;
4231
5540
  const id = auditRecord.recordId;
4232
5541
  if (thisId == null || isDescendantId(thisId, id)) {
4233
- const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.localTime);
5542
+ const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
4234
5543
  if (
4235
5544
  !send({
4236
5545
  id,
4237
- localTime: auditRecord.localTime,
5546
+ localTime: auditRecord.txnLogKey,
4238
5547
  value,
4239
5548
  version: auditRecord.version,
4240
5549
  type: auditRecord.type,
@@ -4247,7 +5556,7 @@ export function makeTable(options) {
4247
5556
  if ((await subscription.waitForDrain()) === false) return;
4248
5557
  }
4249
5558
  }
4250
- subscription!.startTime = auditRecord.localTime ?? auditRecord.version; // update so we don't double send
5559
+ subscription!.startTime = auditRecord.txnLogKey; // update so we don't double send
4251
5560
  }
4252
5561
  } finally {
4253
5562
  // replay is done, we can start sending real-time messages again
@@ -4264,7 +5573,8 @@ export function makeTable(options) {
4264
5573
  if (!isActive()) return;
4265
5574
  }
4266
5575
  try {
4267
- if (auditRecord.tableId !== tableId) continue;
5576
+ if (auditRecord.tableId !== tableId || auditRecord.type === 'evict') continue;
5577
+ if (isLockControlType(auditRecord.type)) continue;
4268
5578
  const id = auditRecord.recordId;
4269
5579
  if (thisId == null || isDescendantId(thisId, id)) {
4270
5580
  // Bound entries INSPECTED for THIS scope, independent of `count` (entries
@@ -4279,10 +5589,10 @@ export function makeTable(options) {
4279
5589
  );
4280
5590
  break;
4281
5591
  }
4282
- const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.localTime);
5592
+ const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
4283
5593
  const historyEntry = {
4284
5594
  id,
4285
- localTime: auditRecord.localTime,
5595
+ localTime: auditRecord.txnLogKey,
4286
5596
  value,
4287
5597
  version: auditRecord.version,
4288
5598
  type: auditRecord.type,
@@ -4296,7 +5606,7 @@ export function makeTable(options) {
4296
5606
  if (--count <= 0) break;
4297
5607
  }
4298
5608
  } catch (error) {
4299
- logger.error?.('Error getting history entry', auditRecord.localTime, error);
5609
+ logger.error?.('Error getting history entry', auditRecord.txnLogKey, error);
4300
5610
  }
4301
5611
  }
4302
5612
  for (let i = history.length; i > 0;) {
@@ -4376,6 +5686,12 @@ export function makeTable(options) {
4376
5686
  logger.trace?.('re-retrieved record', localTime, this.#entry?.localTime);
4377
5687
  localTime = entry?.localTime;
4378
5688
  }
5689
+ let nodeId = entry?.nodeId;
5690
+ if (isRocksDB && entry) {
5691
+ const head = resolveAuditHead(thisId, entry.version, nodeId, entry.additionalAuditRefs);
5692
+ localTime = head.txnLogKey;
5693
+ nodeId = head.nodeId;
5694
+ }
4379
5695
  logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
4380
5696
  if (startTime < localTime) {
4381
5697
  // start time specified, get the audit history for this record. Set startTime up
@@ -4386,7 +5702,6 @@ export function makeTable(options) {
4386
5702
  const history = [];
4387
5703
  let inspected = 0;
4388
5704
  let nextTime = localTime;
4389
- let nodeId = entry?.nodeId;
4390
5705
  do {
4391
5706
  if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
4392
5707
  recordsSinceYield = 0;
@@ -4411,8 +5726,16 @@ export function makeTable(options) {
4411
5726
  if (count) count--;
4412
5727
  } else if (!isActive()) return;
4413
5728
  }
4414
- nextTime = auditRecord.previousVersion;
4415
- nodeId = auditRecord.previousNodeId;
5729
+ const previousHead = isRocksDB
5730
+ ? resolveAuditHead(
5731
+ thisId,
5732
+ auditRecord.previousVersion,
5733
+ auditRecord.previousNodeId,
5734
+ auditRecord.previousAdditionalAuditRefs
5735
+ )
5736
+ : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
5737
+ nextTime = previousHead.txnLogKey;
5738
+ nodeId = previousHead.nodeId;
4416
5739
  } else break;
4417
5740
  } while (nextTime > startTime && count !== 0);
4418
5741
  for (let i = history.length; i > 0;) {
@@ -4591,6 +5914,7 @@ export function makeTable(options) {
4591
5914
  store: primaryStore,
4592
5915
  entry: this.#entry,
4593
5916
  nodeName: (context as any)?.nodeName,
5917
+ recordVersion: options?.version,
4594
5918
  validate: () => {
4595
5919
  if (!(context as any)?.source) {
4596
5920
  transaction.checkOverloaded();
@@ -4675,6 +5999,115 @@ export function makeTable(options) {
4675
5999
  });
4676
6000
  });
4677
6001
  }
6002
+ /**
6003
+ * Write one cluster record-lock control entry (harper#483 Phase 1). Not local-only: replicating
6004
+ * it IS the send.
6005
+ *
6006
+ * `recordId` must stay null. An entry carrying the locked key would share
6007
+ * `(version, tableId, recordId, nodeId)` with the holder's own first write, which is stamped at
6008
+ * exactly `ts_R`, and `RocksTransactionLogStore.getSync` answers with the FIRST entry at a
6009
+ * timestamp and key — so `_writeUpdate`'s keyed dedup would find this one and drop that write.
6010
+ * The payload goes in as bytes rather than through `recordUpdater`, which would run it through
6011
+ * schema projection and the table's shared structure dictionary.
6012
+ */
6013
+ static writeLockControlEntry(entry: LockControlEntry): Promise<number | undefined> {
6014
+ const encodedRecord = encodeLockControlPayload(entry);
6015
+ const nodeId = getThisNodeId(auditStore) ?? 0;
6016
+ let position: number;
6017
+ // No entry pins its clock, the request included. `ts_R` is minted before the write, so pinning
6018
+ // to it can land the entry behind a peer's replication cursor if any write to this table
6019
+ // commits in between — the same hazard that rules it out for grants and releases, which are
6020
+ // written later still. The protocol reads `ts_R` from the payload, so the entry's own log key
6021
+ // never has to equal it.
6022
+ const context = {};
6023
+ return Promise.resolve(
6024
+ transaction(context as any, (txn: any) => {
6025
+ const tableTxn = txnForContext({ transaction: txn } as any);
6026
+ tableTxn.addWrite({
6027
+ key: null,
6028
+ store: primaryStore,
6029
+ skipReplicationConfirmation: true,
6030
+ commit: (txnTime: number, _existingEntry: any, _retry: any, nativeTransaction: any) => {
6031
+ position = txnTime;
6032
+ return auditStore[isRocksDB ? 'putSync' : 'put'](
6033
+ null,
6034
+ {
6035
+ version: txnTime,
6036
+ tableId,
6037
+ recordId: null,
6038
+ nodeId,
6039
+ type: entry.type,
6040
+ encodedRecord,
6041
+ extendedType: 0,
6042
+ // Zero, not the table's count: these bytes were packed by the private control `Packr`
6043
+ // and carry none of the table's structures. `RocksTransactionLogStore` raises the
6044
+ // per-(log, table) structure watermark from this field and flags the entry that does
6045
+ // it, so claiming the table's version would let a release take `HAS_STRUCTURE_UPDATE`
6046
+ // and leave the next real write at that version unflagged — a receiver that learns
6047
+ // structures only from flagged entries then decodes later records against a stale set
6048
+ // (harper#1348's class). A payload with no table structures cannot advance them.
6049
+ structureVersion: 0,
6050
+ },
6051
+ { instructedWrite: true, transaction: nativeTransaction, nodeId, viaNodeId: nodeId }
6052
+ );
6053
+ },
6054
+ });
6055
+ })
6056
+ ).then(() => position);
6057
+ }
6058
+ /**
6059
+ * The coordinator that holds this node's admissions, transport or not. Releasing and registering
6060
+ * go here rather than through `lockCoordinator`, which answers undefined while a transport is
6061
+ * momentarily unregistered — and a release dropped on that answer leaves the key's home holding
6062
+ * its grant until the delegation's own deadline.
6063
+ */
6064
+ static get admittingCoordinator(): LockCoordinator | undefined {
6065
+ return lockCoordinator;
6066
+ }
6067
+
6068
+ /**
6069
+ * This table's cluster lock coordinator, created on first use and only while a transport is
6070
+ * registered for the database. Nothing is allocated on the Phase 0 path.
6071
+ */
6072
+ static get lockCoordinator(): LockCoordinator | undefined {
6073
+ const transport = getClusterLockTransport(databaseName);
6074
+ if (!transport) {
6075
+ // Deliberately NOT closed. harper-pro unregisters without a standalone claim during a
6076
+ // reconnect, and closing here would drop this node's record of the delegations it has
6077
+ // issued as a home — so the next registration would start empty and could grant a key
6078
+ // whose delegate is still admitting. The coordinator keeps ticking, its grants expire on
6079
+ // their own deadlines, and `isClusterLockRequired` is what fails an acquire closed in the
6080
+ // meantime. A genuine standalone claim clears the requirement and the coordinator with it.
6081
+ if (!isClusterLockRequired(databaseName)) {
6082
+ lockCoordinator?.close();
6083
+ lockCoordinator = undefined;
6084
+ }
6085
+ return undefined;
6086
+ }
6087
+ if (lockCoordinator?.transport !== transport) {
6088
+ // The transport object changed, but this node's delegations and the handles they admitted
6089
+ // did not. The successor adopts that live authority in its constructor; the predecessor
6090
+ // is closed afterwards so nothing is dropped in between. See LockCoordinatorOptions.adopt.
6091
+ const predecessor = lockCoordinator;
6092
+ lockCoordinator = new LockCoordinator({
6093
+ database: databaseName,
6094
+ table: tableName,
6095
+ nodeId: getThisNodeName(),
6096
+ transport,
6097
+ adopt: predecessor,
6098
+ // Writing to the local transaction log IS the send, so a transport that only computes
6099
+ // the participant set gets core's writer.
6100
+ writeControl: transport.writeControl
6101
+ ? (entry: LockControlEntry) => transport.writeControl!(tableName, entry)
6102
+ : (entry: LockControlEntry) => TableResource.writeLockControlEntry(entry),
6103
+ keyIdOf: writeKeyId,
6104
+ nextTimestamp: () => (primaryStore as any).getMonotonicTimestamp(),
6105
+ grantableAfterMono: transport.grantableAfterMono,
6106
+ });
6107
+ predecessor?.close();
6108
+ }
6109
+ return lockCoordinator;
6110
+ }
4678
6111
  // #section: validation
4679
6112
  validate(record: any, patch?: boolean) {
4680
6113
  // Accumulate structured per-field issues so the 400 carries `{ path, code,
@@ -4852,6 +6285,7 @@ export function makeTable(options) {
4852
6285
  return this.#version;
4853
6286
  }
4854
6287
  static async addAttributes(attributesToAdd: Attribute[]) {
6288
+ TableResource.assertSchemaMutable('add attributes');
4855
6289
  const new_attributes = attributes.slice(0);
4856
6290
  for (const attribute of attributesToAdd) {
4857
6291
  if (!attribute.name) throw new ClientError('Attribute name is required');
@@ -4869,6 +6303,7 @@ export function makeTable(options) {
4869
6303
  return (TableResource as any).indexingOperation;
4870
6304
  }
4871
6305
  static async removeAttributes(names: string[]) {
6306
+ TableResource.assertSchemaMutable('remove attributes');
4872
6307
  const new_attributes = attributes.filter((attribute) => !names.includes(attribute.name));
4873
6308
  table({
4874
6309
  table: tableName,
@@ -4889,6 +6324,10 @@ export function makeTable(options) {
4889
6324
  const stats = primaryStore.getStats();
4890
6325
  return (stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize;
4891
6326
  }
6327
+ /** Sizes of this table's durable record-structure dictionaries. */
6328
+ static getStructureCounts(): StructureCounts | undefined {
6329
+ return primaryStore.encoder?.getStructureCounts?.();
6330
+ }
4892
6331
  static getAuditSize(): number {
4893
6332
  const stats = auditStore?.getStats();
4894
6333
  return (
@@ -5091,6 +6530,7 @@ export function makeTable(options) {
5091
6530
  // Refresh on every call: schema reload mutates `attributes` in place, so the
5092
6531
  // class-construction snapshot would otherwise go stale.
5093
6532
  this.embedAttributes = (this.attributes as any[]).filter((a) => a?.embed);
6533
+ expiresAtProperty = this.attributes.find((attribute) => attribute.expiresAt);
5094
6534
  // Drop registry entries for attributes that are no longer `@embed`, so a dropped
5095
6535
  // directive doesn't leave a stale embedder or block a default refresh on re-add.
5096
6536
  const embedNames = new Set(this.embedAttributes.map((a) => a.name));
@@ -5160,7 +6600,7 @@ export function makeTable(options) {
5160
6600
  txnForContext(context).getReadTxn(),
5161
6601
  false,
5162
6602
  relatedTable,
5163
- false
6603
+ { allowFullScan: false }
5164
6604
  ) as any
5165
6605
  ).map((entry) => {
5166
6606
  if (entry && entry.key !== undefined) return entry;
@@ -5367,29 +6807,108 @@ export function makeTable(options) {
5367
6807
  this.userSetEmbedders.add(attribute_name);
5368
6808
  }
5369
6809
  static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise<number> {
5370
- let completion: Promise<void>;
6810
+ const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
6811
+ const inFlightRemovals = new Set<Promise<void>>();
6812
+ const removalSlotWaiters: Array<() => void> = [];
6813
+ let removalsAttempted = 0;
6814
+ let removalsSucceeded = 0;
6815
+ let firstRemovalError: unknown;
6816
+ function startRemoval(remove: () => MaybePromise<void>, errorMessage: string, onSuccess?: () => void): void {
6817
+ removalsAttempted++;
6818
+ const removal = new Promise<void>((resolve) => resolve(remove()))
6819
+ .then(
6820
+ () => {
6821
+ removalsSucceeded++;
6822
+ onSuccess?.();
6823
+ },
6824
+ (error) => {
6825
+ // capture before logging: a throwing logger must not cost us the error we may rethrow
6826
+ if (firstRemovalError === undefined) firstRemovalError = error;
6827
+ harperLogger.warn(errorMessage, error);
6828
+ }
6829
+ )
6830
+ .catch(() => undefined)
6831
+ .finally(() => {
6832
+ inFlightRemovals.delete(removal);
6833
+ removalSlotWaiters.shift()?.();
6834
+ });
6835
+ inFlightRemovals.add(removal);
6836
+ }
6837
+ function queueRemoval(
6838
+ remove: () => MaybePromise<void>,
6839
+ errorMessage: string,
6840
+ onSuccess?: () => void
6841
+ ): Promise<void> | undefined {
6842
+ if (inFlightRemovals.size >= maxConcurrentRemovals) {
6843
+ return new Promise<void>((resolve) => {
6844
+ removalSlotWaiters.push(resolve);
6845
+ }).then(() => startRemoval(remove, errorMessage, onSuccess));
6846
+ }
6847
+ startRemoval(remove, errorMessage, onSuccess);
6848
+ }
6849
+ const drainRemovals = () => Promise.all(inFlightRemovals);
5371
6850
  let entriesDeleted = 0;
5372
- for (const auditRecord of auditStore.getRange({
5373
- start: 0,
5374
- end: endTime,
5375
- })) {
5376
- await rest(); // yield to other async operations
5377
- if (auditRecord.tableId !== tableId) continue;
5378
- completion = removeAuditEntry(auditStore, auditRecord);
5379
- entriesDeleted++;
6851
+ // LMDB only: RocksTransactionLogStore.remove() is a no-op, so a RocksDB deleteHistory removes
6852
+ // nothing and must not claim it did.
6853
+ // A bound above everything reachable must not be recorded as the floor: the floor only rises
6854
+ // and a store with a record is never re-stamped, so it would never come down, for every table in
6855
+ // this database. `boundedAuditPruneEnd` clamps the cutoff to just above the newest key in the
6856
+ // log, and the scan below uses that same value as its range end, so the prune cannot remove an
6857
+ // entry the floor does not cover.
6858
+ let pruneEnd = endTime;
6859
+ if (!isRocksDB) {
6860
+ pruneEnd = boundedAuditPruneEnd(auditStore, endTime);
6861
+ raiseAuditFloor(auditStore, pruneEnd);
6862
+ }
6863
+ try {
6864
+ for (const auditRecord of auditStore.getRange({
6865
+ // must not be zero: 0 encodes to all zero bytes and so overlaps the symbol keys, as in
6866
+ // getHistory below
6867
+ start: 1,
6868
+ end: pruneEnd,
6869
+ })) {
6870
+ await rest(); // yield to other async operations
6871
+ if (auditRecord.tableId !== tableId) continue;
6872
+ const backpressure = queueRemoval(
6873
+ () => removeAuditEntry(auditStore, auditRecord),
6874
+ 'Error removing audit entry during deleteHistory',
6875
+ () => {
6876
+ entriesDeleted++;
6877
+ }
6878
+ );
6879
+ if (backpressure) await backpressure;
6880
+ }
6881
+ } finally {
6882
+ await drainRemovals();
5380
6883
  }
5381
6884
  if (cleanupDeletedRecords) {
5382
6885
  // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't
5383
6886
  // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records
5384
- for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
5385
- const { value, localTime } = entry;
5386
- await rest(); // yield to other async operations
5387
- if (value === null && localTime < endTime) {
5388
- completion = removeEntry(primaryStore, entry);
6887
+ try {
6888
+ for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
6889
+ const { key, value, localTime, version } = entry;
6890
+ await rest(); // yield to other async operations
6891
+ const auditTime =
6892
+ isRocksDB && version != null
6893
+ ? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
6894
+ : localTime;
6895
+ if (value === null && version != null && auditTime < pruneEnd) {
6896
+ const backpressure = queueRemoval(
6897
+ () => primaryStore.remove(key, version),
6898
+ 'Error removing deleted record during deleteHistory'
6899
+ );
6900
+ if (backpressure) await backpressure;
6901
+ }
5389
6902
  }
6903
+ } finally {
6904
+ await drainRemovals();
5390
6905
  }
5391
6906
  }
5392
- await completion;
6907
+ if (removalsAttempted > 0 && removalsSucceeded === 0) {
6908
+ // zero progress must not report the same success as "nothing was eligible" (see DESIGN.md);
6909
+ // partial failures stay best-effort, logged and excluded from the returned count
6910
+ throw firstRemovalError ?? new Error('Every removal attempted during deleteHistory failed');
6911
+ }
5393
6912
  return entriesDeleted;
5394
6913
  }
5395
6914
  static async *getHistory(startTime = 0, endTime = Infinity) {
@@ -5398,13 +6917,15 @@ export function makeTable(options) {
5398
6917
  end: endTime,
5399
6918
  })) {
5400
6919
  await rest(); // yield to other async operations
5401
- if (auditRecord.tableId !== tableId) continue;
6920
+ if (auditRecord.tableId !== tableId || auditRecord.type === 'evict' || isLockControlType(auditRecord.type))
6921
+ continue;
5402
6922
  yield {
5403
6923
  id: auditRecord.recordId,
5404
- localTime: auditRecord.version,
6924
+ // Compatibility-facing LMDB history has always reported/grouped by record version.
6925
+ localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
5405
6926
  version: auditRecord.version,
5406
6927
  type: auditRecord.type,
5407
- value: auditRecord.getValue(primaryStore, true, auditRecord.version),
6928
+ value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
5408
6929
  user: auditRecord.user,
5409
6930
  operation: auditRecord.originatingOperation,
5410
6931
  };
@@ -5415,7 +6936,9 @@ export function makeTable(options) {
5415
6936
  if (id == undefined) throw new Error('An id is required');
5416
6937
  const entry = primaryStore.getEntry(id);
5417
6938
  if (!entry) return history;
5418
- let nextVersion = entry.localTime;
6939
+ let nextVersion = isRocksDB
6940
+ ? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
6941
+ : entry.localTime;
5419
6942
  if (!nextVersion) throw new Error('The entry does not have a local audit time');
5420
6943
  const count = 0;
5421
6944
  const auditWindow = 100;
@@ -5425,20 +6948,33 @@ export function makeTable(options) {
5425
6948
  let highestPreviousVersion = 0;
5426
6949
  const start = nextVersion - auditWindow;
5427
6950
  for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) {
5428
- if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) {
6951
+ if (
6952
+ auditRecord.tableId === tableId &&
6953
+ auditRecord.type !== 'evict' &&
6954
+ !isLockControlType(auditRecord.type) &&
6955
+ compareKeys(auditRecord.recordId, id) === 0
6956
+ ) {
5429
6957
  history.splice(insertionPoint, 0, {
5430
6958
  id: auditRecord.recordId,
5431
- localTime: auditRecord.version,
6959
+ localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
5432
6960
  version: auditRecord.version,
5433
6961
  type: auditRecord.type,
5434
- // reconstruct each entry's record image as of its own version, not the audit
6962
+ // reconstruct each entry's record image as of its own log position, not the audit
5435
6963
  // window boundary (nextVersion), matching getHistory (issue #1330)
5436
- value: auditRecord.getValue(primaryStore, true, auditRecord.version),
6964
+ value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
5437
6965
  user: auditRecord.user,
5438
6966
  operation: auditRecord.originatingOperation,
5439
6967
  });
5440
- if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) {
5441
- highestPreviousVersion = auditRecord.previousVersion;
6968
+ const previousVersion = isRocksDB
6969
+ ? resolveAuditHead(
6970
+ id,
6971
+ auditRecord.previousVersion,
6972
+ auditRecord.previousNodeId,
6973
+ auditRecord.previousAdditionalAuditRefs
6974
+ ).txnLogKey
6975
+ : auditRecord.previousVersion;
6976
+ if (previousVersion > highestPreviousVersion && previousVersion < start) {
6977
+ highestPreviousVersion = previousVersion;
5442
6978
  }
5443
6979
  }
5444
6980
  }
@@ -5453,12 +6989,20 @@ export function makeTable(options) {
5453
6989
  const promises = [primaryStore.clear()];
5454
6990
  for (const key in indices) {
5455
6991
  const index = indices[key];
6992
+ index.customIndex?.resetDerivedStorage?.();
5456
6993
  promises.push(index.clearAsync ? index.clearAsync() : index.clear());
5457
6994
  }
5458
6995
  return Promise.all(promises);
5459
6996
  }
6997
+ /** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
5460
6998
  static cleanup() {
6999
+ disposed = true;
7000
+ void TableResource.derivedIndexRuntime?.close();
7001
+ clearTimeout(cleanupTimer);
7002
+ settlePendingCleanup();
7003
+ clearInterval(recordExpirationInterval);
5461
7004
  deleteCallbackHandle?.remove();
7005
+ removeStorageReclamationHandler(primaryStore.path, reclamationHandler);
5462
7006
  }
5463
7007
  static _readTxnForContext(context) {
5464
7008
  return txnForContext(context).getReadTxn();
@@ -5480,9 +7024,21 @@ export function makeTable(options) {
5480
7024
  }
5481
7025
  );
5482
7026
 
5483
- TableResource.updatedAttributes(); // on creation, update accessors as well
5484
- if (expirationMs) TableResource.setTTLExpiration(expirationMs / 1000);
5485
- if (expiresAtProperty) runRecordExpirationEviction();
7027
+ try {
7028
+ TableResource.updatedAttributes(); // on creation, update accessors as well
7029
+ if (expirationMs) {
7030
+ ttlFromLoad = true;
7031
+ try {
7032
+ TableResource.setTTLExpiration(expirationMs / 1000);
7033
+ } finally {
7034
+ ttlFromLoad = false;
7035
+ }
7036
+ }
7037
+ if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction();
7038
+ } catch (error) {
7039
+ TableResource.cleanup();
7040
+ throw error;
7041
+ }
5486
7042
  return TableResource;
5487
7043
  function updateIndices(id: any, existingRecord: any, record: any, options?: any) {
5488
7044
  let hasChanges;
@@ -5889,6 +7445,22 @@ export function makeTable(options) {
5889
7445
  return transaction;
5890
7446
  }
5891
7447
  }
7448
+ /**
7449
+ * Detach an unsaved TransactionWrite that a scoped lock() eagerly staged (see #reloadLocked)
7450
+ * once its handle upgrades to hold: hold staging is deferred and explicit-save-only, so a
7451
+ * dangling scoped write would otherwise auto-commit at the transaction sweep and clobber
7452
+ * whatever the hold write lands. Marking it .dropped lets a later save() on the instance that
7453
+ * owns it (checked via #savingOperation === this write) fall through to the hold branch
7454
+ * instead of resolving a detached, dead reference.
7455
+ */
7456
+ function detachScopedUpgradeWrite(link: any, keyId: unknown, handle: RecordLockHandle): void {
7457
+ for (const write of link.writes) {
7458
+ if (write && !write.saved && write.lockHandle === handle && writeKeyId(write.key) === keyId) {
7459
+ write.dropped = true;
7460
+ link.detachWrite(write);
7461
+ }
7462
+ }
7463
+ }
5892
7464
  function getAttributeValue(entry, attribute_name, context, sort?) {
5893
7465
  if (!entry) {
5894
7466
  return;
@@ -6006,6 +7578,11 @@ export function makeTable(options) {
6006
7578
  const metadataFlags = existingEntry?.metadataFlags;
6007
7579
 
6008
7580
  const existingVersion = existingEntry?.version;
7581
+ const existingRecord = existingEntry?.value;
7582
+ const inheritedTimestamp = context?.timestamp || context?.transaction?.timestamp;
7583
+ const sourceTimestamp =
7584
+ inheritedTimestamp ||
7585
+ (isRocksDB ? (primaryStore as RocksDatabase).getMonotonicTimestamp() : getNextMonotonicTime());
6009
7586
  let whenResolved, timer;
6010
7587
  // We start by locking the record so that there is only one resolution happening at once;
6011
7588
  // if there is already a resolution in process, we want to use the results of that resolution
@@ -6044,10 +7621,8 @@ export function makeTable(options) {
6044
7621
  // lock acquired — this request will actually load from source
6045
7622
  setLoadedFromSource(target, true);
6046
7623
 
6047
- const existingRecord = existingEntry?.value;
6048
7624
  // it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
6049
- // with the canonical data, we are simply fulfilling our local copy of the canonical data, but still don't
6050
- // want a timestamp later than the current transaction
7625
+ // with the canonical data, we are simply fulfilling our local copy of the canonical data.
6051
7626
  // we create a new context for the source, we want to determine the timestamp and don't want to
6052
7627
  // attribute this to the current user
6053
7628
  const sourceContext = {
@@ -6083,13 +7658,37 @@ export function makeTable(options) {
6083
7658
  // before the drain's fail-closed timeout below.
6084
7659
  const commitPromise = transaction(sourceContext, async (_txn) => {
6085
7660
  const start = performance.now();
6086
- let updatedRecord;
7661
+ let updatedRecord, assignCreatedTime, sourceVersion;
6087
7662
  let hasChanges, invalidated;
6088
7663
  try {
6089
7664
  updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
6090
7665
  invalidated = metadataFlags & INVALIDATED;
6091
- let version = sourceContext.lastModified || (invalidated && existingVersion);
6092
- hasChanges = invalidated || version > existingVersion || !existingRecord;
7666
+ const reportedVersion = sourceContext.lastModified;
7667
+ const validReportedVersion =
7668
+ typeof reportedVersion === 'number' &&
7669
+ Number.isFinite(reportedVersion) &&
7670
+ reportedVersion > 0 &&
7671
+ reportedVersion <= MAX_DATE_TIMESTAMP;
7672
+ if (validReportedVersion) {
7673
+ // A record version is also this node's ordering token (precedesExistingVersion), so a
7674
+ // source-reported version ahead of local time would make every subsequent local write look
7675
+ // out-of-order and be discarded until wall-clock caught up — freezing the row. Honor what
7676
+ // the source reports, but never beyond now.
7677
+ const versionCeiling = Math.max(sourceTimestamp, Date.now());
7678
+ sourceVersion = Math.min(reportedVersion, versionCeiling);
7679
+ if (sourceVersion !== reportedVersion) {
7680
+ logger.trace?.(
7681
+ `Capping future source version for ${tableName} id ${id}: ${reportedVersion} -> ${sourceVersion}`
7682
+ );
7683
+ if (!warnedFutureSourceVersion) {
7684
+ warnedFutureSourceVersion = true;
7685
+ logger.warn?.(
7686
+ `The source for ${tableName} reported a lastModified ahead of local time (${new Date(reportedVersion).toISOString()}) for id ${id}; capping cached record versions at local time`
7687
+ );
7688
+ }
7689
+ }
7690
+ } else sourceVersion = sourceTimestamp;
7691
+ hasChanges = invalidated || (validReportedVersion && reportedVersion > existingVersion) || !existingRecord;
6093
7692
  const resolveDuration = performance.now() - start;
6094
7693
  recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success');
6095
7694
  if (responseHeaders)
@@ -6103,7 +7702,7 @@ export function makeTable(options) {
6103
7702
  if (status === 304) {
6104
7703
  // revalidation of our current cached record
6105
7704
  updatedRecord = existingRecord;
6106
- version = existingVersion;
7705
+ sourceVersion = existingVersion;
6107
7706
  } else if (!CACHEABLE_STATUS_CODES.has(status)) {
6108
7707
  // non-cacheable status - propagate to client without caching
6109
7708
  throw new ServerError(updatedRecord.body || 'Error from source', status);
@@ -6173,10 +7772,15 @@ export function makeTable(options) {
6173
7772
  updatedRecord = storedFieldsOnly(primaryStore.encoder, updatedRecord);
6174
7773
  if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
6175
7774
  }
7775
+ assignCreatedTime = createdTimeProperty && updatedRecord?.[createdTimeProperty.name] == null;
6176
7776
  resolved = true;
7777
+ const resolvedVersion =
7778
+ isRocksDB && updatedRecord && existingVersion != null
7779
+ ? Math.max(sourceVersion, existingVersion)
7780
+ : sourceVersion;
6177
7781
  const resolvedEntry: Entry = {
6178
7782
  key: id,
6179
- version,
7783
+ version: resolvedVersion,
6180
7784
  value: updatedRecord,
6181
7785
  expiresAt: sourceContext.expiresAt,
6182
7786
  metadataFlags: 0,
@@ -6242,16 +7846,33 @@ export function makeTable(options) {
6242
7846
  const sourceWrite: any = {
6243
7847
  key: id,
6244
7848
  store: primaryStore,
6245
- entry: existingEntry,
7849
+ entry: undefined,
6246
7850
  nodeName: 'source',
6247
- commit: (txnTime, existingEntry, _retry, transaction: any) => {
7851
+ commit: (_txnTime, existingEntry, _retry, transaction: any) => {
6248
7852
  sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
6249
- if (existingEntry?.version !== existingVersion) {
6250
- // don't do anything if the version has changed
7853
+ const racedVersion = existingEntry?.version;
7854
+ // A first fill may replace a record that raced it only when its candidate version strictly
7855
+ // orders after that record. The comparison has to be replica-independent, so a tie leaves the
7856
+ // raced record in place: precedesExistingVersion() would break the tie with *this* node's
7857
+ // name, and a fill from a shared source has no node identity of its own, so two replicas
7858
+ // resolving the same tie could keep different values at the same version.
7859
+ const replacesRacedRecord = racedVersion == null || sourceVersion > racedVersion;
7860
+ if (
7861
+ racedVersion !== existingVersion &&
7862
+ // Revalidations retain exact-CAS semantics; first fills use deterministic ordering.
7863
+ (existingVersion != null || !updatedRecord || !replacesRacedRecord)
7864
+ ) {
7865
+ logger.trace?.(
7866
+ `Discarding resolved record from source with id: ${id}, source version: ${sourceVersion}, current version: ${racedVersion}`
7867
+ );
6251
7868
  sourceWrite.skipped = true;
6252
7869
  return;
6253
7870
  }
6254
- updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
7871
+ const currentRecord = existingEntry?.value;
7872
+ const recordVersion =
7873
+ isRocksDB && racedVersion != null ? Math.max(sourceVersion, racedVersion) : sourceVersion;
7874
+ const txnLogKey = isRocksDB ? transaction?.getTimestamp?.() : recordVersion;
7875
+ updateIndices(id, currentRecord, updatedRecord, transaction && { transaction });
6255
7876
  if (updatedRecord) {
6256
7877
  if (existingEntry) {
6257
7878
  context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
@@ -6262,22 +7883,22 @@ export function makeTable(options) {
6262
7883
  if (updatedTimeProperty) {
6263
7884
  updatedRecord[updatedTimeProperty.name] =
6264
7885
  updatedTimeProperty.type === 'Date'
6265
- ? new Date(txnTime)
7886
+ ? new Date(recordVersion)
6266
7887
  : updatedTimeProperty.type === 'String'
6267
- ? new Date(txnTime).toISOString()
6268
- : txnTime;
7888
+ ? new Date(recordVersion).toISOString()
7889
+ : recordVersion;
6269
7890
  }
6270
- if (createdTimeProperty && updatedRecord[createdTimeProperty.name] == null) {
6271
- const existingCreatedTime = existingEntry?.value?.[createdTimeProperty.name];
7891
+ if (assignCreatedTime) {
7892
+ const existingCreatedTime = currentRecord?.[createdTimeProperty.name];
6272
7893
  if (existingCreatedTime != null) {
6273
7894
  updatedRecord[createdTimeProperty.name] = existingCreatedTime;
6274
7895
  } else {
6275
7896
  updatedRecord[createdTimeProperty.name] =
6276
7897
  createdTimeProperty.type === 'Date'
6277
- ? new Date(txnTime)
7898
+ ? new Date(recordVersion)
6278
7899
  : createdTimeProperty.type === 'String'
6279
- ? new Date(txnTime).toISOString()
6280
- : txnTime;
7900
+ ? new Date(recordVersion).toISOString()
7901
+ : recordVersion;
6281
7902
  }
6282
7903
  }
6283
7904
  const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
@@ -6309,22 +7930,25 @@ export function makeTable(options) {
6309
7930
  residencyId = getResidencyId(residency);
6310
7931
  }
6311
7932
  logger.trace?.(
6312
- `Writing resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
7933
+ `Writing resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`
6313
7934
  );
6314
7935
  // TODO: We are doing a double check for ifVersion that should probably be cleaned out
7936
+ const writeAudit = (audit && (hasChanges || omitLocalRecord)) || null;
6315
7937
  updateRecord(
6316
7938
  id,
6317
7939
  updatedRecord,
6318
7940
  existingEntry,
6319
- txnTime,
7941
+ recordVersion,
6320
7942
  omitLocalRecord ? INVALIDATED : 0,
6321
- (audit && (hasChanges || omitLocalRecord)) || null,
7943
+ writeAudit,
6322
7944
  {
6323
7945
  user: (sourceContext as any)?.user,
6324
7946
  expiresAt: sourceContext.expiresAt,
6325
7947
  residencyId,
6326
7948
  transaction,
6327
7949
  tableToTrack: tableName,
7950
+ additionalAuditRefs:
7951
+ writeAudit && txnLogKey !== recordVersion ? [{ version: txnLogKey, nodeId: 0 }] : undefined,
6328
7952
  },
6329
7953
  'put',
6330
7954
  Boolean(invalidated),
@@ -6334,17 +7958,26 @@ export function makeTable(options) {
6334
7958
  if (sourceContext.expiresAt) scheduleCleanup();
6335
7959
  } else if (existingEntry) {
6336
7960
  logger.trace?.(
6337
- `Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
7961
+ `Deleting resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`
6338
7962
  );
6339
7963
  if (audit || trackDeletes) {
6340
7964
  updateRecord(
6341
7965
  id,
6342
7966
  null,
6343
7967
  existingEntry,
6344
- txnTime,
7968
+ recordVersion,
6345
7969
  0,
6346
7970
  (audit && hasChanges) || null,
6347
- { user: (sourceContext as any)?.user, transaction, tableToTrack: tableName },
7971
+ {
7972
+ user: (sourceContext as any)?.user,
7973
+ transaction,
7974
+ tableToTrack: tableName,
7975
+ recordVersion,
7976
+ additionalAuditRefs:
7977
+ audit && hasChanges && txnLogKey !== recordVersion
7978
+ ? [{ version: txnLogKey, nodeId: 0 }]
7979
+ : undefined,
7980
+ },
6348
7981
  'delete',
6349
7982
  Boolean(invalidated)
6350
7983
  );
@@ -6433,6 +8066,7 @@ export function makeTable(options) {
6433
8066
  if (entry.value == null) continue; // already removed
6434
8067
  if (hasSourceGet && primaryStore.hasLock(item.key, entry.version)) continue; // resolution in progress
6435
8068
  updateIndices(item.key, entry.value, null, options);
8069
+ stageDerivedIndexEviction(transaction, item.key, entry.version);
6436
8070
  }
6437
8071
  removeEntry(primaryStore, entry, options);
6438
8072
  staged++;
@@ -6507,7 +8141,13 @@ export function makeTable(options) {
6507
8141
  },
6508
8142
  };
6509
8143
  }
8144
+ function settlePendingCleanup() {
8145
+ for (const resolve of pendingCleanupResolvers) resolve();
8146
+ pendingCleanupResolvers.clear();
8147
+ }
6510
8148
  function scheduleCleanup(priority?: number): Promise<void> | void {
8149
+ // a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
8150
+ if (disposed) return;
6511
8151
  let runImmediately = false;
6512
8152
  if (priority) {
6513
8153
  // run immediately if there is a big increase in priority
@@ -6517,11 +8157,21 @@ export function makeTable(options) {
6517
8157
  // Periodically evict expired records and deleted records searching for records who expiresAt timestamp is before now
6518
8158
  if (cleanupInterval === lastCleanupInterval && !runImmediately) return;
6519
8159
  lastCleanupInterval = cleanupInterval;
6520
- if (getWorkerIndex() === getWorkerCount() - 1) {
8160
+ if (ownsStoreMaintenance(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker())) {
6521
8161
  // run on the last thread so we aren't overloading lower-numbered threads
6522
8162
  if (cleanupTimer) clearTimeout(cleanupTimer);
6523
- if (!cleanupInterval) return;
6524
- return new Promise((resolve) => {
8163
+ if (!cleanupInterval) {
8164
+ // no replacement pass is being scheduled, so nothing is left to settle a superseded one
8165
+ settlePendingCleanup();
8166
+ return;
8167
+ }
8168
+ // This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
8169
+ // this pass's scan completes, so a reclamation run is never told the storage was reclaimed
8170
+ // before any scan ran. It has to run now, though — that run blocks its whole path on the
8171
+ // promise, and the replacement's own slot can be a full interval out.
8172
+ if (pendingCleanupResolvers.size > 0) runImmediately = true;
8173
+ return new Promise<void>((resolve) => {
8174
+ pendingCleanupResolvers.add(resolve);
6525
8175
  const startOfYear = new Date();
6526
8176
  startOfYear.setMonth(0);
6527
8177
  startOfYear.setDate(1);
@@ -6534,6 +8184,7 @@ export function makeTable(options) {
6534
8184
  ? Date.now()
6535
8185
  : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
6536
8186
  const startNextTimer = (nextScheduled) => {
8187
+ if (disposed) return;
6537
8188
  logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
6538
8189
  // noinspection JSVoidFunctionReturnValueUsed
6539
8190
  cleanupTimer = setTimeout(
@@ -6544,8 +8195,11 @@ export function makeTable(options) {
6544
8195
  const rootStore = primaryStore.rootStore;
6545
8196
  if (rootStore.status !== 'open') {
6546
8197
  clearTimeout(cleanupTimer);
8198
+ settlePendingCleanup();
6547
8199
  return;
6548
8200
  }
8201
+ // snapshot: an awaiter that arrives during this scan belongs to the pass that supersedes it
8202
+ const settling = [...pendingCleanupResolvers];
6549
8203
  const MAX_CLEANUP_CONCURRENCY = 50;
6550
8204
  const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY);
6551
8205
  let cleanupIndex = 0;
@@ -6625,7 +8279,10 @@ export function makeTable(options) {
6625
8279
  } catch (error) {
6626
8280
  logger.warn?.(`Error in cleanup scan for ${tableName}:`, error);
6627
8281
  }
6628
- resolve(undefined);
8282
+ for (const settle of settling) {
8283
+ pendingCleanupResolvers.delete(settle);
8284
+ settle();
8285
+ }
6629
8286
  cleanupPriority = 0; // reset the priority
6630
8287
  })),
6631
8288
  Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number
@@ -6637,17 +8294,19 @@ export function makeTable(options) {
6637
8294
  }
6638
8295
  function addDeleteRemoval() {
6639
8296
  deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id: Id, version: number) => {
6640
- primaryStore.remove(id, version);
8297
+ return primaryStore.remove(id, version);
6641
8298
  });
6642
8299
  }
6643
8300
  function runRecordExpirationEviction() {
6644
8301
  // Periodically evict expired records, searching for records who expiresAt timestamp is before now
6645
- if (getWorkerIndex() === 0) {
8302
+ if (ownsStoreExpiration(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker())) {
6646
8303
  // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
6647
- setInterval(async () => {
8304
+ recordExpirationInterval = setInterval(async () => {
6648
8305
  // go through each database and table and then search for expired entries
6649
8306
  // find any entries that are set to expire before now
6650
- if (runningRecordExpiration) return;
8307
+ // updatedAttributes() clears expiresAtProperty when a live redeclaration drops the directive,
8308
+ // and there is nothing left for this interval to scan by
8309
+ if (disposed || runningRecordExpiration || !expiresAtProperty) return;
6651
8310
  runningRecordExpiration = true;
6652
8311
  try {
6653
8312
  const expiresAtName = expiresAtProperty.name;