@harperfast/harper 5.2.9 → 5.3.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (668) hide show
  1. package/agent/mcpTools.ts +1 -1
  2. package/agent/session.ts +25 -14
  3. package/bin/cliOperations.ts +46 -9
  4. package/bin/copyDb.ts +282 -59
  5. package/bin/deploySetup.ts +16 -5
  6. package/bin/harper.ts +1 -1
  7. package/bin/help.ts +4 -1
  8. package/bin/lite.ts +4 -1
  9. package/bin/restart.ts +59 -3
  10. package/bin/run.ts +6 -11
  11. package/bin/upgrade.js +7 -3
  12. package/bin/workloadIdentity.ts +119 -0
  13. package/components/Application.ts +2414 -242
  14. package/components/ApplicationScope.ts +8 -0
  15. package/components/EntryHandler.ts +59 -39
  16. package/components/OptionsWatcher.ts +150 -74
  17. package/components/RuntimeModuleTracker.ts +38 -7
  18. package/components/Scope.ts +37 -15
  19. package/components/awaitRestart.ts +84 -0
  20. package/components/componentLoader.ts +340 -30
  21. package/components/componentPreparationLock.ts +16 -5
  22. package/components/mcp/adapters/harperHttp.ts +4 -0
  23. package/components/mcp/listChanged.ts +4 -0
  24. package/components/mcp/toolRegistry.ts +2 -0
  25. package/components/mcp/tools/operations.ts +9 -0
  26. package/components/mcp/tools/schemas/operationDescriptions.ts +2 -2
  27. package/components/operations.js +270 -78
  28. package/components/operationsValidation.js +49 -1
  29. package/components/status/ComponentStatusRegistry.ts +59 -0
  30. package/config/RootConfigWatcher.ts +80 -34
  31. package/config/configUtils.ts +291 -34
  32. package/config/harperConfigEnvVars.ts +170 -27
  33. package/config-root.schema.json +29 -0
  34. package/dataLayer/blobBackup.ts +160 -50
  35. package/dataLayer/delete.ts +6 -1
  36. package/dataLayer/harperBridge/ResourceBridge.ts +52 -8
  37. package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  38. package/dataLayer/hdbInfoController.ts +34 -1
  39. package/dataLayer/insert.ts +44 -1
  40. package/dataLayer/rocksdbBackup.ts +53 -10
  41. package/dataLayer/schema.ts +11 -1
  42. package/dataLayer/schemaDescribe.ts +8 -1
  43. package/dist/agent/mcpTools.js +1 -1
  44. package/dist/agent/mcpTools.js.map +1 -1
  45. package/dist/agent/session.d.ts +22 -0
  46. package/dist/agent/session.js +26 -15
  47. package/dist/agent/session.js.map +1 -1
  48. package/dist/bin/cliOperations.js +48 -9
  49. package/dist/bin/cliOperations.js.map +1 -1
  50. package/dist/bin/copyDb.d.ts +12 -1
  51. package/dist/bin/copyDb.js +248 -60
  52. package/dist/bin/copyDb.js.map +1 -1
  53. package/dist/bin/deploySetup.d.ts +2 -0
  54. package/dist/bin/deploySetup.js +11 -3
  55. package/dist/bin/deploySetup.js.map +1 -1
  56. package/dist/bin/harper.js +1 -1
  57. package/dist/bin/harper.js.map +1 -1
  58. package/dist/bin/help.js +4 -1
  59. package/dist/bin/help.js.map +1 -1
  60. package/dist/bin/lite.js +4 -1
  61. package/dist/bin/lite.js.map +1 -1
  62. package/dist/bin/restart.js +54 -5
  63. package/dist/bin/restart.js.map +1 -1
  64. package/dist/bin/run.js +4 -10
  65. package/dist/bin/run.js.map +1 -1
  66. package/dist/bin/upgrade.js +4 -3
  67. package/dist/bin/upgrade.js.map +1 -1
  68. package/dist/bin/workloadIdentity.d.ts +18 -0
  69. package/dist/bin/workloadIdentity.js +100 -0
  70. package/dist/bin/workloadIdentity.js.map +1 -0
  71. package/dist/components/Application.d.ts +139 -16
  72. package/dist/components/Application.js +2215 -267
  73. package/dist/components/Application.js.map +1 -1
  74. package/dist/components/ApplicationScope.d.ts +8 -0
  75. package/dist/components/ApplicationScope.js +7 -0
  76. package/dist/components/ApplicationScope.js.map +1 -1
  77. package/dist/components/EntryHandler.js +26 -10
  78. package/dist/components/EntryHandler.js.map +1 -1
  79. package/dist/components/OptionsWatcher.d.ts +1 -0
  80. package/dist/components/OptionsWatcher.js +141 -74
  81. package/dist/components/OptionsWatcher.js.map +1 -1
  82. package/dist/components/RuntimeModuleTracker.js +40 -6
  83. package/dist/components/RuntimeModuleTracker.js.map +1 -1
  84. package/dist/components/Scope.js +38 -13
  85. package/dist/components/Scope.js.map +1 -1
  86. package/dist/components/awaitRestart.d.ts +33 -0
  87. package/dist/components/awaitRestart.js +61 -0
  88. package/dist/components/awaitRestart.js.map +1 -0
  89. package/dist/components/componentLoader.d.ts +38 -1
  90. package/dist/components/componentLoader.js +279 -22
  91. package/dist/components/componentLoader.js.map +1 -1
  92. package/dist/components/componentPreparationLock.d.ts +5 -0
  93. package/dist/components/componentPreparationLock.js +14 -6
  94. package/dist/components/componentPreparationLock.js.map +1 -1
  95. package/dist/components/mcp/adapters/harperHttp.js +4 -0
  96. package/dist/components/mcp/adapters/harperHttp.js.map +1 -1
  97. package/dist/components/mcp/listChanged.js +5 -0
  98. package/dist/components/mcp/listChanged.js.map +1 -1
  99. package/dist/components/mcp/toolRegistry.d.ts +1 -0
  100. package/dist/components/mcp/toolRegistry.js.map +1 -1
  101. package/dist/components/mcp/tools/operations.d.ts +5 -0
  102. package/dist/components/mcp/tools/operations.js +9 -0
  103. package/dist/components/mcp/tools/operations.js.map +1 -1
  104. package/dist/components/mcp/tools/schemas/operationDescriptions.js +2 -2
  105. package/dist/components/mcp/tools/schemas/operationDescriptions.js.map +1 -1
  106. package/dist/components/operations.js +231 -77
  107. package/dist/components/operations.js.map +1 -1
  108. package/dist/components/operationsValidation.js +49 -1
  109. package/dist/components/operationsValidation.js.map +1 -1
  110. package/dist/components/status/ComponentStatusRegistry.d.ts +0 -4
  111. package/dist/components/status/ComponentStatusRegistry.js +63 -0
  112. package/dist/components/status/ComponentStatusRegistry.js.map +1 -1
  113. package/dist/config/RootConfigWatcher.d.ts +7 -1
  114. package/dist/config/RootConfigWatcher.js +64 -27
  115. package/dist/config/RootConfigWatcher.js.map +1 -1
  116. package/dist/config/configUtils.d.ts +9 -1
  117. package/dist/config/configUtils.js +254 -33
  118. package/dist/config/configUtils.js.map +1 -1
  119. package/dist/config/harperConfigEnvVars.d.ts +16 -0
  120. package/dist/config/harperConfigEnvVars.js +162 -25
  121. package/dist/config/harperConfigEnvVars.js.map +1 -1
  122. package/dist/dataLayer/blobBackup.d.ts +49 -20
  123. package/dist/dataLayer/blobBackup.js +139 -50
  124. package/dist/dataLayer/blobBackup.js.map +1 -1
  125. package/dist/dataLayer/delete.js +1 -1
  126. package/dist/dataLayer/delete.js.map +1 -1
  127. package/dist/dataLayer/harperBridge/ResourceBridge.d.ts +14 -1
  128. package/dist/dataLayer/harperBridge/ResourceBridge.js +51 -10
  129. package/dist/dataLayer/harperBridge/ResourceBridge.js.map +1 -1
  130. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.d.ts +3 -1
  131. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js +3 -1
  132. package/dist/dataLayer/harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js.map +1 -1
  133. package/dist/dataLayer/hdbInfoController.d.ts +10 -0
  134. package/dist/dataLayer/hdbInfoController.js +30 -1
  135. package/dist/dataLayer/hdbInfoController.js.map +1 -1
  136. package/dist/dataLayer/insert.d.ts +9 -1
  137. package/dist/dataLayer/insert.js +30 -0
  138. package/dist/dataLayer/insert.js.map +1 -1
  139. package/dist/dataLayer/rocksdbBackup.d.ts +2 -2
  140. package/dist/dataLayer/rocksdbBackup.js +45 -8
  141. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  142. package/dist/dataLayer/schema.js +8 -0
  143. package/dist/dataLayer/schema.js.map +1 -1
  144. package/dist/dataLayer/schemaDescribe.js +8 -1
  145. package/dist/dataLayer/schemaDescribe.js.map +1 -1
  146. package/dist/index.d.ts +1 -1
  147. package/dist/index.js.map +1 -1
  148. package/dist/json/systemSchema.json +52 -0
  149. package/dist/resources/DatabaseTransaction.d.ts +55 -0
  150. package/dist/resources/DatabaseTransaction.js +459 -45
  151. package/dist/resources/DatabaseTransaction.js.map +1 -1
  152. package/dist/resources/LMDBTransaction.js +22 -4
  153. package/dist/resources/LMDBTransaction.js.map +1 -1
  154. package/dist/resources/PrimaryRocksDatabase.d.ts +1 -0
  155. package/dist/resources/PrimaryRocksDatabase.js +29 -0
  156. package/dist/resources/PrimaryRocksDatabase.js.map +1 -1
  157. package/dist/resources/RecordEncoder.d.ts +20 -0
  158. package/dist/resources/RecordEncoder.js +94 -9
  159. package/dist/resources/RecordEncoder.js.map +1 -1
  160. package/dist/resources/RequestTarget.d.ts +2 -0
  161. package/dist/resources/RequestTarget.js.map +1 -1
  162. package/dist/resources/Resource.js +20 -11
  163. package/dist/resources/Resource.js.map +1 -1
  164. package/dist/resources/ResourceInterface.d.ts +20 -1
  165. package/dist/resources/ResourceInterface.js.map +1 -1
  166. package/dist/resources/RocksIndexStore.d.ts +6 -1
  167. package/dist/resources/RocksIndexStore.js +24 -9
  168. package/dist/resources/RocksIndexStore.js.map +1 -1
  169. package/dist/resources/RocksTransactionLogStore.d.ts +14 -1
  170. package/dist/resources/RocksTransactionLogStore.js +57 -17
  171. package/dist/resources/RocksTransactionLogStore.js.map +1 -1
  172. package/dist/resources/Table.d.ts +112 -8
  173. package/dist/resources/Table.js +1073 -191
  174. package/dist/resources/Table.js.map +1 -1
  175. package/dist/resources/auditStore.d.ts +11 -2
  176. package/dist/resources/auditStore.js +325 -71
  177. package/dist/resources/auditStore.js.map +1 -1
  178. package/dist/resources/blob.d.ts +129 -9
  179. package/dist/resources/blob.js +992 -114
  180. package/dist/resources/blob.js.map +1 -1
  181. package/dist/resources/branchDatabase.d.ts +48 -0
  182. package/dist/resources/branchDatabase.js +892 -0
  183. package/dist/resources/branchDatabase.js.map +1 -0
  184. package/dist/resources/crdt.js +50 -12
  185. package/dist/resources/crdt.js.map +1 -1
  186. package/dist/resources/databases.d.ts +150 -0
  187. package/dist/resources/databases.js +1200 -239
  188. package/dist/resources/databases.js.map +1 -1
  189. package/dist/resources/defineTable.d.ts +10 -2
  190. package/dist/resources/defineTable.js +9 -1
  191. package/dist/resources/defineTable.js.map +1 -1
  192. package/dist/resources/graphql.d.ts +1 -1
  193. package/dist/resources/graphql.js +50 -15
  194. package/dist/resources/graphql.js.map +1 -1
  195. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.d.ts +1 -0
  196. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js +6 -2
  197. package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js.map +1 -1
  198. package/dist/resources/longLivedTransactions.d.ts +71 -0
  199. package/dist/resources/longLivedTransactions.js +358 -0
  200. package/dist/resources/longLivedTransactions.js.map +1 -0
  201. package/dist/resources/models/backendRegistry.d.ts +26 -0
  202. package/dist/resources/models/backendRegistry.js +60 -2
  203. package/dist/resources/models/backendRegistry.js.map +1 -1
  204. package/dist/resources/models/bootstrap.d.ts +33 -1
  205. package/dist/resources/models/bootstrap.js +416 -31
  206. package/dist/resources/models/bootstrap.js.map +1 -1
  207. package/dist/resources/recordLock.d.ts +80 -0
  208. package/dist/resources/recordLock.js +184 -0
  209. package/dist/resources/recordLock.js.map +1 -0
  210. package/dist/resources/replayLogs.d.ts +14 -1
  211. package/dist/resources/replayLogs.js +152 -24
  212. package/dist/resources/replayLogs.js.map +1 -1
  213. package/dist/resources/replayLogsGuards.d.ts +94 -7
  214. package/dist/resources/replayLogsGuards.js +111 -7
  215. package/dist/resources/replayLogsGuards.js.map +1 -1
  216. package/dist/resources/search.js +158 -6
  217. package/dist/resources/search.js.map +1 -1
  218. package/dist/resources/transactionBroadcast.d.ts +1 -1
  219. package/dist/resources/transactionBroadcast.js +2 -2
  220. package/dist/resources/transactionBroadcast.js.map +1 -1
  221. package/dist/security/auth.js +61 -30
  222. package/dist/security/auth.js.map +1 -1
  223. package/dist/security/authn/oidc/claims.d.ts +22 -0
  224. package/dist/security/authn/oidc/claims.js +71 -0
  225. package/dist/security/authn/oidc/claims.js.map +1 -0
  226. package/dist/security/authn/oidc/identityToken.d.ts +27 -0
  227. package/dist/security/authn/oidc/identityToken.js +111 -0
  228. package/dist/security/authn/oidc/identityToken.js.map +1 -0
  229. package/dist/security/authn/oidc/jwks.d.ts +25 -0
  230. package/dist/security/authn/oidc/jwks.js +261 -0
  231. package/dist/security/authn/oidc/jwks.js.map +1 -0
  232. package/dist/security/authn/oidc/providers/generic.d.ts +13 -0
  233. package/dist/security/authn/oidc/providers/generic.js +34 -0
  234. package/dist/security/authn/oidc/providers/generic.js.map +1 -0
  235. package/dist/security/authn/oidc/providers/githubActions.d.ts +11 -0
  236. package/dist/security/authn/oidc/providers/githubActions.js +129 -0
  237. package/dist/security/authn/oidc/providers/githubActions.js.map +1 -0
  238. package/dist/security/authn/oidc/providers/index.d.ts +37 -0
  239. package/dist/security/authn/oidc/providers/index.js +24 -0
  240. package/dist/security/authn/oidc/providers/index.js.map +1 -0
  241. package/dist/security/authn/oidc/tokenExchange.d.ts +12 -0
  242. package/dist/security/authn/oidc/tokenExchange.js +306 -0
  243. package/dist/security/authn/oidc/tokenExchange.js.map +1 -0
  244. package/dist/security/authn/oidc/trustPolicyOperations.d.ts +49 -0
  245. package/dist/security/authn/oidc/trustPolicyOperations.js +358 -0
  246. package/dist/security/authn/oidc/trustPolicyOperations.js.map +1 -0
  247. package/dist/security/authn/oidc/types.d.ts +38 -0
  248. package/dist/security/authn/oidc/types.js +6 -0
  249. package/dist/security/authn/oidc/types.js.map +1 -0
  250. package/dist/security/certificateVerification/index.js +40 -11
  251. package/dist/security/certificateVerification/index.js.map +1 -1
  252. package/dist/security/certificateVerification/trustedIssuers.d.ts +24 -0
  253. package/dist/security/certificateVerification/trustedIssuers.js +79 -0
  254. package/dist/security/certificateVerification/trustedIssuers.js.map +1 -0
  255. package/dist/security/certificateVerification/types.d.ts +1 -0
  256. package/dist/security/credentialProvenance.d.ts +35 -0
  257. package/dist/security/credentialProvenance.js +51 -0
  258. package/dist/security/credentialProvenance.js.map +1 -0
  259. package/dist/security/credentialRejection.d.ts +4 -0
  260. package/dist/security/credentialRejection.js +24 -0
  261. package/dist/security/credentialRejection.js.map +1 -0
  262. package/dist/security/deferredAuthentication.d.ts +36 -0
  263. package/dist/security/deferredAuthentication.js +70 -0
  264. package/dist/security/deferredAuthentication.js.map +1 -0
  265. package/dist/security/impersonation.d.ts +21 -0
  266. package/dist/security/impersonation.js +108 -9
  267. package/dist/security/impersonation.js.map +1 -1
  268. package/dist/security/jsLoader.d.ts +6 -0
  269. package/dist/security/jsLoader.js +75 -15
  270. package/dist/security/jsLoader.js.map +1 -1
  271. package/dist/security/keys.js +301 -71
  272. package/dist/security/keys.js.map +1 -1
  273. package/dist/security/operationScope.d.ts +21 -0
  274. package/dist/security/operationScope.js +36 -0
  275. package/dist/security/operationScope.js.map +1 -0
  276. package/dist/security/permissionsTranslator.js +21 -0
  277. package/dist/security/permissionsTranslator.js.map +1 -1
  278. package/dist/security/tokenAuthentication.d.ts +19 -1
  279. package/dist/security/tokenAuthentication.js +191 -10
  280. package/dist/security/tokenAuthentication.js.map +1 -1
  281. package/dist/security/user.js +4 -3
  282. package/dist/security/user.js.map +1 -1
  283. package/dist/server/DurableSubscriptionsSession.d.ts +2 -2
  284. package/dist/server/DurableSubscriptionsSession.js +67 -10
  285. package/dist/server/DurableSubscriptionsSession.js.map +1 -1
  286. package/dist/server/REST.js +73 -0
  287. package/dist/server/REST.js.map +1 -1
  288. package/dist/server/graphqlQuerying.js +4 -0
  289. package/dist/server/graphqlQuerying.js.map +1 -1
  290. package/dist/server/http.d.ts +11 -0
  291. package/dist/server/http.js +88 -15
  292. package/dist/server/http.js.map +1 -1
  293. package/dist/server/itc/serverHandlers.js +8 -1
  294. package/dist/server/itc/serverHandlers.js.map +1 -1
  295. package/dist/server/jobs/jobProcess.js +6 -2
  296. package/dist/server/jobs/jobProcess.js.map +1 -1
  297. package/dist/server/jobs/jobs.js +4 -1
  298. package/dist/server/jobs/jobs.js.map +1 -1
  299. package/dist/server/liveSubscriptionAuth.d.ts +26 -4
  300. package/dist/server/liveSubscriptionAuth.js +105 -39
  301. package/dist/server/liveSubscriptionAuth.js.map +1 -1
  302. package/dist/server/loadRootComponents.js +49 -10
  303. package/dist/server/loadRootComponents.js.map +1 -1
  304. package/dist/server/mqtt.d.ts +2 -0
  305. package/dist/server/mqtt.js +165 -30
  306. package/dist/server/mqtt.js.map +1 -1
  307. package/dist/server/nodeName.d.ts +2 -0
  308. package/dist/server/nodeName.js +107 -23
  309. package/dist/server/nodeName.js.map +1 -1
  310. package/dist/server/serverHelpers/Headers.d.ts +25 -0
  311. package/dist/server/serverHelpers/Headers.js +139 -1
  312. package/dist/server/serverHelpers/Headers.js.map +1 -1
  313. package/dist/server/serverHelpers/contentTypes.d.ts +9 -0
  314. package/dist/server/serverHelpers/contentTypes.js +32 -25
  315. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  316. package/dist/server/serverHelpers/deployValidationState.d.ts +3 -0
  317. package/dist/server/serverHelpers/deployValidationState.js +9 -19
  318. package/dist/server/serverHelpers/deployValidationState.js.map +1 -1
  319. package/dist/server/serverHelpers/operationAuthorizationState.d.ts +12 -0
  320. package/dist/server/serverHelpers/operationAuthorizationState.js +24 -2
  321. package/dist/server/serverHelpers/operationAuthorizationState.js.map +1 -1
  322. package/dist/server/serverHelpers/registeredOperations.d.ts +5 -4
  323. package/dist/server/serverHelpers/registeredOperations.js +74 -21
  324. package/dist/server/serverHelpers/registeredOperations.js.map +1 -1
  325. package/dist/server/serverHelpers/requestSanitization.d.ts +11 -0
  326. package/dist/server/serverHelpers/requestSanitization.js +20 -0
  327. package/dist/server/serverHelpers/requestSanitization.js.map +1 -0
  328. package/dist/server/serverHelpers/serverHandlers.js +6 -3
  329. package/dist/server/serverHelpers/serverHandlers.js.map +1 -1
  330. package/dist/server/serverHelpers/serverUtilities.d.ts +18 -0
  331. package/dist/server/serverHelpers/serverUtilities.js +90 -20
  332. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  333. package/dist/server/serverHelpers/sharedMessageEncoding.d.ts +67 -0
  334. package/dist/server/serverHelpers/sharedMessageEncoding.js +280 -0
  335. package/dist/server/serverHelpers/sharedMessageEncoding.js.map +1 -0
  336. package/dist/server/serverHelpers/uwsServer.js +19 -1
  337. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  338. package/dist/server/static.js +24 -28
  339. package/dist/server/static.js.map +1 -1
  340. package/dist/server/storageReclamation.d.ts +12 -0
  341. package/dist/server/storageReclamation.js +26 -0
  342. package/dist/server/storageReclamation.js.map +1 -1
  343. package/dist/server/threads/itc.d.ts +7 -2
  344. package/dist/server/threads/itc.js +5 -1
  345. package/dist/server/threads/itc.js.map +1 -1
  346. package/dist/server/threads/manageThreads.d.ts +26 -3
  347. package/dist/server/threads/manageThreads.js +517 -65
  348. package/dist/server/threads/manageThreads.js.map +1 -1
  349. package/dist/server/threads/socketRouter.js +89 -26
  350. package/dist/server/threads/socketRouter.js.map +1 -1
  351. package/dist/server/threads/threadHeapMemory.d.ts +2 -0
  352. package/dist/server/threads/threadHeapMemory.js +31 -0
  353. package/dist/server/threads/threadHeapMemory.js.map +1 -0
  354. package/dist/server/threads/threadServer.js +46 -15
  355. package/dist/server/threads/threadServer.js.map +1 -1
  356. package/dist/sqlEngine/config.d.ts +1 -3
  357. package/dist/sqlEngine/config.js +19 -16
  358. package/dist/sqlEngine/config.js.map +1 -1
  359. package/dist/sqlTranslator/index.d.ts +1 -1
  360. package/dist/sqlTranslator/index.js +30 -7
  361. package/dist/sqlTranslator/index.js.map +1 -1
  362. package/dist/upgrade/directives/5-3-0.d.ts +7 -0
  363. package/dist/upgrade/directives/5-3-0.js +148 -0
  364. package/dist/upgrade/directives/5-3-0.js.map +1 -0
  365. package/dist/upgrade/directives/directivesController.js +2 -1
  366. package/dist/upgrade/directives/directivesController.js.map +1 -1
  367. package/dist/utility/OperationFunctionCaller.js +2 -1
  368. package/dist/utility/OperationFunctionCaller.js.map +1 -1
  369. package/dist/utility/common_utils.d.ts +16 -0
  370. package/dist/utility/common_utils.js +32 -6
  371. package/dist/utility/common_utils.js.map +1 -1
  372. package/dist/utility/componentNames.d.ts +8 -0
  373. package/dist/utility/componentNames.js +12 -1
  374. package/dist/utility/componentNames.js.map +1 -1
  375. package/dist/utility/environment/environmentManager.js +3 -6
  376. package/dist/utility/environment/environmentManager.js.map +1 -1
  377. package/dist/utility/environment/systemInformation.d.ts +1 -0
  378. package/dist/utility/environment/systemInformation.js +1 -0
  379. package/dist/utility/environment/systemInformation.js.map +1 -1
  380. package/dist/utility/errors/commonErrors.d.ts +2 -0
  381. package/dist/utility/errors/commonErrors.js +2 -0
  382. package/dist/utility/errors/commonErrors.js.map +1 -1
  383. package/dist/utility/errors/hdbError.d.ts +17 -0
  384. package/dist/utility/errors/hdbError.js +30 -1
  385. package/dist/utility/errors/hdbError.js.map +1 -1
  386. package/dist/utility/globalSchema.d.ts +18 -0
  387. package/dist/utility/hdbTerms.d.ts +16 -0
  388. package/dist/utility/hdbTerms.js +18 -2
  389. package/dist/utility/hdbTerms.js.map +1 -1
  390. package/dist/utility/logging/harper_logger.d.ts +2 -0
  391. package/dist/utility/logging/harper_logger.js +91 -14
  392. package/dist/utility/logging/harper_logger.js.map +1 -1
  393. package/dist/utility/logging/logRotator.js +76 -49
  394. package/dist/utility/logging/logRotator.js.map +1 -1
  395. package/dist/utility/nodeIdentity.d.ts +9 -0
  396. package/dist/utility/nodeIdentity.js +58 -0
  397. package/dist/utility/nodeIdentity.js.map +1 -0
  398. package/dist/utility/npmUtilities.js +9 -7
  399. package/dist/utility/npmUtilities.js.map +1 -1
  400. package/dist/utility/operationPermissions.d.ts +3 -1
  401. package/dist/utility/operationPermissions.js +16 -1
  402. package/dist/utility/operationPermissions.js.map +1 -1
  403. package/dist/utility/operation_authorization.d.ts +10 -7
  404. package/dist/utility/operation_authorization.js +212 -41
  405. package/dist/utility/operation_authorization.js.map +1 -1
  406. package/dist/utility/watchPath.d.ts +29 -0
  407. package/dist/utility/watchPath.js +68 -0
  408. package/dist/utility/watchPath.js.map +1 -0
  409. package/dist/utility/watcherFallback.d.ts +86 -0
  410. package/dist/utility/watcherFallback.js +278 -1
  411. package/dist/utility/watcherFallback.js.map +1 -1
  412. package/dist/validation/configValidator.d.ts +12 -0
  413. package/dist/validation/configValidator.js +199 -72
  414. package/dist/validation/configValidator.js.map +1 -1
  415. package/dist/validation/installValidator.js +12 -0
  416. package/dist/validation/installValidator.js.map +1 -1
  417. package/dist/validation/validationWrapper.d.ts +11 -0
  418. package/dist/validation/validationWrapper.js +16 -3
  419. package/dist/validation/validationWrapper.js.map +1 -1
  420. package/index.ts +2 -0
  421. package/json/systemSchema.json +52 -0
  422. package/npm-shrinkwrap.json +196 -194
  423. package/package.json +7 -6
  424. package/resources/DESIGN.md +114 -52
  425. package/resources/DatabaseTransaction.ts +521 -43
  426. package/resources/LMDBTransaction.ts +26 -4
  427. package/resources/PrimaryRocksDatabase.ts +28 -1
  428. package/resources/RecordEncoder.ts +103 -8
  429. package/resources/RequestTarget.ts +2 -0
  430. package/resources/Resource.ts +17 -9
  431. package/resources/ResourceInterface.ts +23 -0
  432. package/resources/RocksIndexStore.ts +30 -9
  433. package/resources/RocksTransactionLogStore.ts +78 -21
  434. package/resources/Table.ts +1110 -159
  435. package/resources/auditStore.ts +331 -73
  436. package/resources/blob.ts +1029 -110
  437. package/resources/branchDatabase.ts +941 -0
  438. package/resources/crdt.ts +76 -12
  439. package/resources/databases.ts +1310 -252
  440. package/resources/defineTable.ts +18 -2
  441. package/resources/graphql.ts +70 -16
  442. package/resources/indexes/HierarchicalNavigableSmallWorld.ts +6 -2
  443. package/resources/longLivedTransactions.ts +360 -0
  444. package/resources/models/backendRegistry.ts +84 -2
  445. package/resources/models/bootstrap.ts +473 -28
  446. package/resources/recordLock.ts +253 -0
  447. package/resources/replayLogs.ts +147 -26
  448. package/resources/replayLogsGuards.ts +171 -8
  449. package/resources/search.ts +154 -7
  450. package/resources/transactionBroadcast.ts +3 -3
  451. package/security/auth.ts +68 -29
  452. package/security/authn/oidc/claims.ts +72 -0
  453. package/security/authn/oidc/identityToken.ts +129 -0
  454. package/security/authn/oidc/jwks.ts +260 -0
  455. package/security/authn/oidc/providers/generic.ts +40 -0
  456. package/security/authn/oidc/providers/githubActions.ts +137 -0
  457. package/security/authn/oidc/providers/index.ts +52 -0
  458. package/security/authn/oidc/tokenExchange.ts +300 -0
  459. package/security/authn/oidc/trustPolicyOperations.ts +343 -0
  460. package/security/authn/oidc/types.ts +41 -0
  461. package/security/certificateVerification/index.ts +54 -13
  462. package/security/certificateVerification/trustedIssuers.ts +76 -0
  463. package/security/certificateVerification/types.ts +1 -0
  464. package/security/credentialProvenance.ts +47 -0
  465. package/security/credentialRejection.ts +22 -0
  466. package/security/deferredAuthentication.ts +71 -0
  467. package/security/impersonation.ts +117 -12
  468. package/security/jsLoader.ts +81 -18
  469. package/security/keys.ts +298 -72
  470. package/security/operationScope.ts +33 -0
  471. package/security/permissionsTranslator.js +23 -0
  472. package/security/tokenAuthentication.ts +233 -12
  473. package/security/user.ts +4 -3
  474. package/server/DESIGN.md +183 -16
  475. package/server/DurableSubscriptionsSession.ts +71 -11
  476. package/server/REST.ts +79 -1
  477. package/server/graphqlQuerying.ts +4 -0
  478. package/server/http.ts +99 -18
  479. package/server/itc/serverHandlers.js +8 -1
  480. package/server/jobs/jobProcess.ts +8 -2
  481. package/server/jobs/jobs.ts +4 -1
  482. package/server/liveSubscriptionAuth.ts +129 -46
  483. package/server/loadRootComponents.js +50 -8
  484. package/server/mqtt.ts +179 -38
  485. package/server/nodeName.ts +103 -21
  486. package/server/serverHelpers/Headers.ts +131 -0
  487. package/server/serverHelpers/contentTypes.ts +29 -21
  488. package/server/serverHelpers/deployValidationState.ts +24 -13
  489. package/server/serverHelpers/operationAuthorizationState.ts +34 -3
  490. package/server/serverHelpers/registeredOperations.ts +79 -22
  491. package/server/serverHelpers/requestSanitization.ts +15 -0
  492. package/server/serverHelpers/serverHandlers.js +6 -3
  493. package/server/serverHelpers/serverUtilities.ts +104 -31
  494. package/server/serverHelpers/sharedMessageEncoding.ts +307 -0
  495. package/server/serverHelpers/uwsServer.ts +17 -2
  496. package/server/static.ts +23 -29
  497. package/server/storageReclamation.ts +23 -0
  498. package/server/threads/itc.js +11 -1
  499. package/server/threads/manageThreads.js +526 -56
  500. package/server/threads/socketRouter.ts +81 -26
  501. package/server/threads/threadHeapMemory.ts +26 -0
  502. package/server/threads/threadServer.js +43 -15
  503. package/sqlTranslator/index.ts +31 -8
  504. package/studio/web/assets/Chat-BnCBegQz.js +2267 -0
  505. package/studio/web/assets/FloatingChat-CoDW1ySS.js +23 -0
  506. package/studio/web/assets/{abnfDiagram-VCTEODGH-C0_BAZyO.js → abnfDiagram-VCTEODGH-B0BebmD2.js} +1 -1
  507. package/studio/web/assets/{alertDialog-DIHt7Z0r.js → alertDialog-CQyAJJhl.js} +1 -1
  508. package/studio/web/assets/{apiToken-c3Rd-w6g.js → apiToken-Bwk5BLXW.js} +1 -1
  509. package/studio/web/assets/applications-DHxGi7JH.js +296 -0
  510. package/studio/web/assets/architecture-7GRP2DOG-LB-MLAAb.js +1 -0
  511. package/studio/web/assets/{architectureDiagram-5GKGNRK7-BWzrASgm.js → architectureDiagram-5GKGNRK7-7SW3GD-K.js} +1 -1
  512. package/studio/web/assets/authStore-C3Nfubqr.js +3 -0
  513. package/studio/web/assets/{blockDiagram-NRAW4CY4-BdJX9Khj.js → blockDiagram-I7D4REHJ-BqguiadH.js} +2 -2
  514. package/studio/web/assets/{button-DhiX-njv.js → button-BIsUKRZq.js} +2 -2
  515. package/studio/web/assets/{c4Diagram-UCG6FXSJ-CI6MzGmQ.js → c4Diagram-7LVT6UL2-LBNf8t_X.js} +1 -1
  516. package/studio/web/assets/channel-yictG-U-.js +1 -0
  517. package/studio/web/assets/{chevron-up-Df2c8uoU.js → chevron-up-DtKGqDn3.js} +1 -1
  518. package/studio/web/assets/{chunk-TEH6E4GO-P87k5mNi.js → chunk-4HAMMTFA-DWtTut21.js} +1 -1
  519. package/studio/web/assets/{chunk-75Z2AOVW-BT8tVmks.js → chunk-75Z2AOVW-QGQD6th2.js} +1 -1
  520. package/studio/web/assets/{chunk-DU6HZSFF-9kAOOmI6.js → chunk-DU6HZSFF-Chq20Ba5.js} +1 -1
  521. package/studio/web/assets/{chunk-F27PBJKO-BW7ao8AY.js → chunk-F27PBJKO-BVA5EPhV.js} +1 -1
  522. package/studio/web/assets/{chunk-GMAD6QVW-BNyXpoQO.js → chunk-GMAD6QVW-BeS7S07A.js} +1 -1
  523. package/studio/web/assets/{chunk-OBVCFTLP-D4wWOqDQ.js → chunk-GVQU2GXP-sbwVIQ8i.js} +1 -1
  524. package/studio/web/assets/{chunk-G27WJ6UU-COyLMcgK.js → chunk-IMKFNOWR-Bnh3tAVd.js} +1 -1
  525. package/studio/web/assets/{chunk-JQ64N6SF-Cyz1IeLf.js → chunk-L3NEJ4N5-COfUyKII.js} +1 -1
  526. package/studio/web/assets/chunk-OSK3NFVY-ByciRftO.js +10 -0
  527. package/studio/web/assets/{chunk-P2QGCYS3-DmIFY4d7.js → chunk-P2QGCYS3-CP1VhG_c.js} +1 -1
  528. package/studio/web/assets/{chunk-POPQ4Y6H-BPrvMyKz.js → chunk-POPQ4Y6H-ClWhhkwW.js} +1 -1
  529. package/studio/web/assets/{chunk-PWAF6VOD-2zB6IW9i.js → chunk-PWAF6VOD-1z1THyS5.js} +1 -1
  530. package/studio/web/assets/{chunk-RHFEMEQ7-2FgyI8YU.js → chunk-SHT3W25Y-LpQkMsah.js} +2 -2
  531. package/studio/web/assets/{chunk-SVP7TREG-FwtbH2QC.js → chunk-SVP7TREG-jtdAHw0S.js} +1 -1
  532. package/studio/web/assets/{chunk-LCL6LL3I-HOzK_ppE.js → chunk-TICWLB2K-VOwzetX-.js} +1 -1
  533. package/studio/web/assets/classDiagram-ZZMXUADV-VaEwSy_g.js +1 -0
  534. package/studio/web/assets/classDiagram-v2-VYDZK3BY-VaEwSy_g.js +1 -0
  535. package/studio/web/assets/{createLucideIcon-BKGPfjm2.js → createLucideIcon-CzW9508A.js} +1 -1
  536. package/studio/web/assets/{cssMode-CEN2mzSA.js → cssMode-s0cWI-_M.js} +1 -1
  537. package/studio/web/assets/{cynefin-OW5HDTMX-BRkpLFQV.js → cynefin-OW5HDTMX-BbdbCvub.js} +1 -1
  538. package/studio/web/assets/{cynefinDiagram-5FMLGOSQ-CHT1DaX6.js → cynefinDiagram-5FMLGOSQ-TP-aIqbt.js} +1 -1
  539. package/studio/web/assets/{dagre-3AP2YEHR-DpUXBh63.js → dagre-GXQ25YYZ-DShnGpGo.js} +1 -1
  540. package/studio/web/assets/{diagram-S7CK7UJ4-BuymVFZT.js → diagram-S7CK7UJ4-aoCVTtcy.js} +1 -1
  541. package/studio/web/assets/{diagram-UQ7AKVKN-CyP148RM.js → diagram-UQ7AKVKN-DglXtQ6x.js} +1 -1
  542. package/studio/web/assets/{diagram-VSXAHHWV-CoCAg3M9.js → diagram-VSXAHHWV-fhEdmkwM.js} +1 -1
  543. package/studio/web/assets/{diagram-VX7I27RA-BpOqCFca.js → diagram-VX7I27RA-DccVJet6.js} +1 -1
  544. package/studio/web/assets/{diagram-Z3DM3KII-Bfpw7Vbj.js → diagram-Z3DM3KII-D-RyJJb7.js} +1 -1
  545. package/studio/web/assets/{dialog-CBf0Mr1d.js → dialog-Cn2uWgD4.js} +1 -1
  546. package/studio/web/assets/{dist-lkA3O3eM.js → dist-DP8UjMB_.js} +1 -1
  547. package/studio/web/assets/{download-BtTOBem-.js → download-B5T5r7ss.js} +1 -1
  548. package/studio/web/assets/{ebnfDiagram-PWID7BFC-DS_6aWqL.js → ebnfDiagram-PWID7BFC-DJGpIpz_.js} +1 -1
  549. package/studio/web/assets/{editor-D8oDeCTL.js → editor-DNcRHK54.js} +1 -1
  550. package/studio/web/assets/{erDiagram-SSCWMZ5O-DJNk6Fgw.js → erDiagram-RLTQ6QDP-CIfNlgkC.js} +1 -1
  551. package/studio/web/assets/eventmodeling-NTZA5JFV-CLxnp2CR.js +1 -0
  552. package/studio/web/assets/flowDiagram-HODETNUW-BIbhmz9f.js +1 -0
  553. package/studio/web/assets/{ganttDiagram-EL5Y4UJY-2pOExxMY.js → ganttDiagram-EL5Y4UJY-BxToTzzD.js} +1 -1
  554. package/studio/web/assets/{getAnalytics-D4LKGeVy.js → getAnalytics-GHK8ORfM.js} +1 -1
  555. package/studio/web/assets/{gitGraph-4MIJSDKK-CH5ZxwzF.js → gitGraph-4MIJSDKK-D2s2w8lE.js} +1 -1
  556. package/studio/web/assets/{gitGraphDiagram-WWUBYQGX-DVIsIhbO.js → gitGraphDiagram-WWUBYQGX-Dwntd4-x.js} +1 -1
  557. package/studio/web/assets/{html-u3vOg7LJ.js → html-Bdssedlg.js} +1 -1
  558. package/studio/web/assets/{htmlMode-DyO31v-P.js → htmlMode-CoDlJ3fw.js} +1 -1
  559. package/studio/web/assets/{index-Cxj2_wsl.css → index-7RMEgVG1.css} +1 -1
  560. package/studio/web/assets/index-D6sxmFLR.js +824 -0
  561. package/studio/web/assets/index.lazy-tmU5BS8s.js +2 -0
  562. package/studio/web/assets/{info-A6RAGUB7-CPQfTnaG.js → info-A6RAGUB7-DYjkvb0C.js} +1 -1
  563. package/studio/web/assets/{infoDiagram-RXCK75RN-DlwLYlwm.js → infoDiagram-27XIBGKW-Bnp1FJE5.js} +1 -1
  564. package/studio/web/assets/{ishikawaDiagram-5VMMS53U-BRXRp29U.js → ishikawaDiagram-5VMMS53U-D9Xh2r6X.js} +1 -1
  565. package/studio/web/assets/{javascript-CUvxOyTC.js → javascript-B8meVSTH.js} +1 -1
  566. package/studio/web/assets/{journeyDiagram-EYS64GPL-B0ou8k0n.js → journeyDiagram-3NMN7TZE-CokIi6ll.js} +2 -2
  567. package/studio/web/assets/{jsonMode-f_IwbF3D.js → jsonMode-DpIPd35T.js} +1 -1
  568. package/studio/web/assets/{kanban-definition-3QL26DDD-uYg7iYzp.js → kanban-definition-UXKFOSKX-CukSFJfX.js} +1 -1
  569. package/studio/web/assets/{languageServices-DXtZ6rEF.js → languageServices-C_5FMJzQ.js} +1 -1
  570. package/studio/web/assets/{lspLanguageFeatures-B4pCF1zO.js → lspLanguageFeatures-BIzNBzPK.js} +1 -1
  571. package/studio/web/assets/{mermaid-parser.core-Ck-fC8b7.js → mermaid-parser.core-BlEsOWNO.js} +3 -3
  572. package/studio/web/assets/{mermaid.core-CP8aNNYm.js → mermaid.core-BlkGaMIH.js} +5 -5
  573. package/studio/web/assets/{mindmap-definition-FBJOCRG2-CgTZ-rit.js → mindmap-definition-YA3MSWOX-IprMc_0j.js} +1 -1
  574. package/studio/web/assets/{notifications-D3tIQ4sg.js → notifications-CQf18QKb.js} +1 -1
  575. package/studio/web/assets/notifications-CvZivSbh.js +1 -0
  576. package/studio/web/assets/{packet-AYTQ26CC-DEyoPtPb.js → packet-AYTQ26CC-Bi3V04Zi.js} +1 -1
  577. package/studio/web/assets/{pegDiagram-XKGWAZYB-DrD-7sD9.js → pegDiagram-XKGWAZYB-BNuPDLZY.js} +1 -1
  578. package/studio/web/assets/{pie-WAS4IAKB-wjj-EI1d.js → pie-WAS4IAKB-_6DoDbng.js} +1 -1
  579. package/studio/web/assets/{pieDiagram-E7YTZNPT-GntqDCzv.js → pieDiagram-E7YTZNPT-DqNb6Ht2.js} +1 -1
  580. package/studio/web/assets/{profile-DZWU7MgT.js → profile-DdOwtntb.js} +1 -1
  581. package/studio/web/assets/{quadrantDiagram-AXDQQJYC-0UeqXQGd.js → quadrantDiagram-AXDQQJYC-BGH9E2YR.js} +1 -1
  582. package/studio/web/assets/{radar-RG4KPBEZ-DFSA5h7k.js → radar-RG4KPBEZ-DDdVczcL.js} +1 -1
  583. package/studio/web/assets/{railroad-74A4TZTK-CaOUG9wR.js → railroad-74A4TZTK-BJUP4Jds.js} +1 -1
  584. package/studio/web/assets/railroad-abnf-HS5TGJTU-Bm3L1L0h.js +1 -0
  585. package/studio/web/assets/railroad-ebnf-LZEXJU2U-CKzLGlkw.js +1 -0
  586. package/studio/web/assets/railroad-peg-WCYAUIDC-S8xLjslx.js +1 -0
  587. package/studio/web/assets/{railroadDiagram-O6MQD6OU-yHUELZaV.js → railroadDiagram-O6MQD6OU-DGTPh2KZ.js} +1 -1
  588. package/studio/web/assets/{regions-CkyurXzE.js → regions-n69fwagr.js} +1 -1
  589. package/studio/web/assets/{register-BUyhWjBO.js → register-PfWTCXWB.js} +3 -3
  590. package/studio/web/assets/{requirementDiagram-EFPCY7ZU-DNEGFjuW.js → requirementDiagram-BXWQKSXE-BJnO6uLz.js} +1 -1
  591. package/studio/web/assets/{sankeyDiagram-P5KCCOFB-DpyAmSVR.js → sankeyDiagram-P5KCCOFB-0vSOdymH.js} +1 -1
  592. package/studio/web/assets/{sequenceDiagram-WJ2MYXX4-TyaT7xNk.js → sequenceDiagram-WJ2MYXX4-hETizDWE.js} +1 -1
  593. package/studio/web/assets/{setComponentFile-CeyKSZAa.js → setComponentFile-Bg6O7X0S.js} +1 -1
  594. package/studio/web/assets/{setup-D2kn7cAA.js → setup-CUx_aUDl.js} +2 -2
  595. package/studio/web/assets/{stateDiagram-HBIQ2CUA-CeEdTArZ.js → stateDiagram-D77RDMKH-CdYQ_KtC.js} +1 -1
  596. package/studio/web/assets/stateDiagram-v2-MP3YSRHH-CdKuzQMT.js +1 -0
  597. package/studio/web/assets/status-D7BVKqX9.js +61 -0
  598. package/studio/web/assets/{swimlanes-XN3QIQJK-B54FmF46.js → swimlanes-42K2YHIH-B8cHIpU4.js} +1 -1
  599. package/studio/web/assets/swimlanesDiagram-VR7AAH4N-DmOSJwaH.js +8 -0
  600. package/studio/web/assets/{tabs-B_G5zscN.js → tabs-BrHu7gJi.js} +1 -1
  601. package/studio/web/assets/{timeline-definition-24CTP7MA-D-a9ujbo.js → timeline-definition-24CTP7MA-BJWYSXqF.js} +1 -1
  602. package/studio/web/assets/{toggleHighContrast-C0UW6rI2.js → toggleHighContrast-DBSyXzMr.js} +1 -1
  603. package/studio/web/assets/{treeView-Q6P3EWNA-CrW_6JnS.js → treeView-Q6P3EWNA-qxe_v6CQ.js} +1 -1
  604. package/studio/web/assets/{treemap-WGGIJYW6-BxyYLdP_.js → treemap-WGGIJYW6-dDo97XXF.js} +1 -1
  605. package/studio/web/assets/{tsMode-DBC0zmDx.js → tsMode-BByKCjBS.js} +1 -1
  606. package/studio/web/assets/{typescript-DApRQir3.js → typescript-DDLnLpw9.js} +1 -1
  607. package/studio/web/assets/{useEntityRestURL-DB6JStU1.js → useEntityRestURL-31CHGaHk.js} +1 -1
  608. package/studio/web/assets/{useLocalStorage-Dtj1QS8_.js → useLocalStorage-BqMR3D8_.js} +1 -1
  609. package/studio/web/assets/vendor-core-c2JRRJpV.js +58 -0
  610. package/studio/web/assets/vendor-datadog-CLUcJXOo.js +6 -0
  611. package/studio/web/assets/{vendor-react-Dyj4O3HE.js → vendor-react-CJV_K1u4.js} +1 -1
  612. package/studio/web/assets/vendor-tanstack-DxzraizX.js +1 -0
  613. package/studio/web/assets/{vendor-ui-vhu-UHhF.js → vendor-ui-BUjK0h8a.js} +2 -2
  614. package/studio/web/assets/{vennDiagram-4TSXK5OY-Cy7s7Mpy.js → vennDiagram-4TSXK5OY-A3i-lCdl.js} +1 -1
  615. package/studio/web/assets/{wardley-WFR3VGLG-BeBL35g2.js → wardley-WFR3VGLG-B0ik-_6g.js} +1 -1
  616. package/studio/web/assets/{wardleyDiagram-VM6X3IG4-BylmIGSg.js → wardleyDiagram-VM6X3IG4-CjrkKWUR.js} +1 -1
  617. package/studio/web/assets/{workers-C0bFIedw.js → workers-pR3jRY9D.js} +1 -1
  618. package/studio/web/assets/x-DIzaLEdK.js +1 -0
  619. package/studio/web/assets/{xml-HWd01lU-.js → xml-2iRnMhQO.js} +1 -1
  620. package/studio/web/assets/{xychartDiagram-S5SC5T6Z-CoKALMXr.js → xychartDiagram-S5SC5T6Z-Biok4GYV.js} +1 -1
  621. package/studio/web/assets/{yaml-CIH0Nt-h.js → yaml-Bf92gJpd.js} +1 -1
  622. package/studio/web/index.html +14 -14
  623. package/upgrade/directives/5-3-0.ts +132 -0
  624. package/upgrade/directives/directivesController.ts +2 -1
  625. package/utility/OperationFunctionCaller.ts +2 -1
  626. package/utility/common_utils.ts +30 -5
  627. package/utility/componentNames.ts +12 -0
  628. package/utility/environment/environmentManager.ts +3 -7
  629. package/utility/environment/systemInformation.ts +7 -0
  630. package/utility/errors/commonErrors.ts +4 -0
  631. package/utility/errors/hdbError.ts +29 -0
  632. package/utility/hdbTerms.ts +17 -0
  633. package/utility/logging/harper_logger.ts +87 -14
  634. package/utility/logging/logRotator.ts +72 -45
  635. package/utility/nodeIdentity.ts +45 -0
  636. package/utility/npmUtilities.ts +10 -8
  637. package/utility/operationPermissions.ts +18 -1
  638. package/utility/operation_authorization.ts +231 -42
  639. package/utility/watchPath.ts +63 -0
  640. package/utility/watcherFallback.ts +270 -0
  641. package/validation/configValidator.ts +209 -72
  642. package/validation/installValidator.ts +15 -0
  643. package/validation/validationWrapper.ts +18 -4
  644. package/studio/web/assets/Chat-JpO8EtUu.js +0 -2067
  645. package/studio/web/assets/FloatingChat-Bcj3xSZu.js +0 -23
  646. package/studio/web/assets/applications-ByqLRKyZ.js +0 -296
  647. package/studio/web/assets/architecture-7GRP2DOG-DNdx5tEU.js +0 -1
  648. package/studio/web/assets/authStore-qKmCZcaf.js +0 -3
  649. package/studio/web/assets/channel-DtCV8PTL.js +0 -1
  650. package/studio/web/assets/chunk-R7TYR2AO-Irip67yr.js +0 -10
  651. package/studio/web/assets/classDiagram-DTDB5LWJ-DbO_dCNE.js +0 -1
  652. package/studio/web/assets/classDiagram-v2-JRS7N3AN-DbO_dCNE.js +0 -1
  653. package/studio/web/assets/eventmodeling-NTZA5JFV-5jbe4A5P.js +0 -1
  654. package/studio/web/assets/flowDiagram-A5DVABFB-Dp9Ezlow.js +0 -1
  655. package/studio/web/assets/index-aSt5tY-L.js +0 -824
  656. package/studio/web/assets/index.lazy-B9jiPwT8.js +0 -2
  657. package/studio/web/assets/notifications-CUoYgU98.js +0 -1
  658. package/studio/web/assets/railroad-abnf-HS5TGJTU-Bc0Qi0WH.js +0 -1
  659. package/studio/web/assets/railroad-ebnf-LZEXJU2U-G8rVVZ2C.js +0 -1
  660. package/studio/web/assets/railroad-peg-WCYAUIDC-CrehKBhC.js +0 -1
  661. package/studio/web/assets/stateDiagram-v2-4QOOHH4V-D4tuw9Su.js +0 -1
  662. package/studio/web/assets/status-D7Xn5ePA.js +0 -61
  663. package/studio/web/assets/swimlanesDiagram-VK2B7HYN-XOhmNEvq.js +0 -8
  664. package/studio/web/assets/vendor-core-RCcadM3e.js +0 -73
  665. package/studio/web/assets/vendor-datadog-BRv-mOv1.js +0 -6
  666. package/studio/web/assets/vendor-tanstack-BiFWSB3W.js +0 -1
  667. package/studio/web/assets/x-B9o9hsep.js +0 -1
  668. /package/studio/web/assets/{sizeCapture-X5ZJPWSS-B0uUizjq.js → sizeCapture-INFHLROL-B0uUizjq.js} +0 -0
@@ -14,7 +14,8 @@ import {
14
14
  import { type Database } from 'lmdb';
15
15
  import { Script } from 'node:vm';
16
16
  import { randomUUID } from 'node:crypto';
17
- import { getIndexedValues } from '../utility/lmdb/commonUtility.ts';
17
+ import { performance } from 'node:perf_hooks';
18
+ import { getIndexedValues, getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
18
19
  import { getThisNodeId, exportIdMapping } from './nodeIdMapping.ts';
19
20
  import lodash from 'lodash';
20
21
  import { ExtendedIterable, SKIP } from '@harperfast/extended-iterable';
@@ -41,6 +42,14 @@ import {
41
42
  TRANSACTION_STATE,
42
43
  writeKeyId,
43
44
  } from './DatabaseTransaction.ts';
45
+ import {
46
+ acquireRecordKey,
47
+ lockAttemptKey,
48
+ lockNotHeldError,
49
+ resolveLockOptions,
50
+ type RecordLockHandle,
51
+ type RecordLockOptions,
52
+ } from './recordLock.ts';
44
53
  import * as envMngr from '../utility/environment/environmentManager.ts';
45
54
  import { addSubscription } from './transactionBroadcast.ts';
46
55
  import {
@@ -49,6 +58,7 @@ import {
49
58
  ServerError,
50
59
  AccessViolation,
51
60
  ValidationError,
61
+ UpdateAttributesLockTimeoutError,
52
62
  type ValidationIssue,
53
63
  } from '../utility/errors/hdbError.ts';
54
64
  import * as signalling from '../utility/signalling.ts';
@@ -58,6 +68,7 @@ import {
58
68
  searchByIndex,
59
69
  findAttribute,
60
70
  estimateCondition,
71
+ estimatedEntryCount,
61
72
  flattenKey,
62
73
  COERCIBLE_OPERATORS,
63
74
  executeConditions,
@@ -78,6 +89,7 @@ import {
78
89
  PENDING_LOCAL_TIME,
79
90
  RecordObject,
80
91
  type Entry,
92
+ type StructureCounts,
81
93
  entryMap,
82
94
  storedFieldsOnly,
83
95
  } from './RecordEncoder.ts';
@@ -86,7 +98,12 @@ import { rebuildUpdateBefore } from './crdt.ts';
86
98
  import { appendHeader } from '../server/serverHelpers/Headers.ts';
87
99
  import fs from 'node:fs';
88
100
  import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts';
89
- import { onStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts';
101
+ import {
102
+ onStorageReclamation,
103
+ removeStorageReclamation,
104
+ removeStorageReclamationHandler,
105
+ getStorageSpaceStats,
106
+ } from '../server/storageReclamation.ts';
90
107
  import { RequestTarget } from './RequestTarget.ts';
91
108
  import harperLogger from '../utility/logging/harper_logger.ts';
92
109
  import { throttle } from '../server/throttle.ts';
@@ -130,7 +147,10 @@ type MaybePromise<T> = T | Promise<T>;
130
147
  const NULL_WITH_TIMESTAMP = new Uint8Array(9);
131
148
  NULL_WITH_TIMESTAMP[8] = 0xc0; // null
132
149
  const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
150
+ const MAX_DATE_TIMESTAMP = 8.64e15;
133
151
  const RECORD_PRUNING_INTERVAL = 60000; // one minute
152
+ const MAX_CONCURRENT_HISTORY_REMOVALS = 10;
153
+ const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
134
154
  // RocksDB-only: number of eviction/tombstone removals coalesced into a single transaction commit.
135
155
  // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
136
156
  // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
@@ -139,9 +159,93 @@ const EVICTION_BATCH_SIZE = 100;
139
159
  // letting an unbounded number of open transactions (and their snapshots) accumulate.
140
160
  const MAX_INFLIGHT_EVICTION_BATCHES = 4;
141
161
  const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
162
+ // Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
163
+ // of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
164
+ // into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
165
+ // truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
166
+ // (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
167
+ const MAX_EXACT_COUNT_SCAN = 1_000_000;
168
+ const MAX_EXACT_COUNT_MS = 1_000;
169
+ // Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
170
+ // not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
171
+ // streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
172
+ const MAX_COUNT_PAGE = 10_000;
173
+ // How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
174
+ // check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
175
+ const COUNT_YIELD_INTERVAL = 2_048;
142
176
  envMngr.initSync();
143
177
  const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
144
178
  const LOCK_TIMEOUT = 10000;
179
+ // This bounds schema-lock acquisition; LOCK_TIMEOUT bounds in-flight record writes during a drop.
180
+ export const UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000;
181
+ const UPDATE_ATTRIBUTES_LOCK = 'update-attributes';
182
+ // Contention is otherwise only visible once it becomes a timeout (harper#2251).
183
+ export const UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT = 1000;
184
+ // raw ASCII bytes are ordered-binary's encoding of the string, so this addresses the same native
185
+ // lock as string-keyed tryLock/unlock calls
186
+ const updateAttributesLockKey = Buffer.from(UPDATE_ATTRIBUTES_LOCK);
187
+ const lockWait = new Int32Array(new SharedArrayBuffer(4));
188
+
189
+ /** The wait blocks the event loop, so the locked section must stay synchronous. */
190
+ export function acquireUpdateAttributesLock(
191
+ rootStore: RocksDatabase,
192
+ scopeDescription: string,
193
+ timeout = UPDATE_ATTRIBUTES_LOCK_TIMEOUT
194
+ ) {
195
+ if (rootStore.tryLock(updateAttributesLockKey)) return;
196
+ const startTime = performance.now();
197
+ let waitTime = 1;
198
+ while (!rootStore.tryLock(updateAttributesLockKey)) {
199
+ const elapsed = performance.now() - startTime;
200
+ if (elapsed >= timeout) {
201
+ throw new UpdateAttributesLockTimeoutError(
202
+ `Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`
203
+ );
204
+ }
205
+ if (elapsed >= 2) {
206
+ Atomics.wait(lockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
207
+ if (waitTime < 16) waitTime *= 2;
208
+ }
209
+ }
210
+ const waited = performance.now() - startTime;
211
+ // The caller cannot register its release until we return, so a throw here would leak the lock
212
+ // with no `finally` able to reach it.
213
+ if (waited >= UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
214
+ try {
215
+ logger.warn?.(
216
+ `Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms; this worker's event loop was blocked for that wait, and a holder that runs past ${UPDATE_ATTRIBUTES_LOCK_TIMEOUT}ms fails the update outright`
217
+ );
218
+ } catch {}
219
+ }
220
+
221
+ export function releaseUpdateAttributesLock(rootStore: RocksDatabase) {
222
+ rootStore.unlock(updateAttributesLockKey);
223
+ }
224
+
225
+ export function withUpdateAttributesLock<Callback extends () => unknown>(
226
+ rootStore: RocksDatabase,
227
+ scopeDescription: string,
228
+ callback: Callback & (ReturnType<Callback> extends PromiseLike<unknown> ? never : unknown)
229
+ ): ReturnType<Callback> {
230
+ acquireUpdateAttributesLock(rootStore, scopeDescription);
231
+ try {
232
+ const result = callback();
233
+ if (typeof (result as any)?.then === 'function') {
234
+ Promise.resolve(result).catch((error) =>
235
+ logger.error?.(
236
+ `Async update-attributes callback rejected after its lock was released (${scopeDescription})`,
237
+ error
238
+ )
239
+ );
240
+ throw new TypeError(
241
+ `withUpdateAttributesLock callback must be synchronous (${scopeDescription}); asynchronous work may continue after the lock is released`
242
+ );
243
+ }
244
+ return result as ReturnType<Callback>;
245
+ } finally {
246
+ releaseUpdateAttributesLock(rootStore);
247
+ }
248
+ }
145
249
  // Tolerate a redundant column family drop. Drops are broadcast to every worker
146
250
  // thread and each holds its own handle to the same underlying family, so a
147
251
  // concurrent worker may already have dropped it; the storage engine reports
@@ -380,6 +484,24 @@ function chainKeyForId(id: any): string {
380
484
  return typeof id === 'string' ? 's' + id : 'k' + writeKeyId(id);
381
485
  }
382
486
 
487
+ /** Normalizes a passed `context` argument as `transactional()` does; undefined means fall back to ambient. */
488
+ function contextArgument(context: unknown): any {
489
+ if (!context || isReleasedTransaction(context)) return undefined;
490
+ const resolved = (context as any).getContext?.() || context;
491
+ return resolved instanceof DatabaseTransaction ? { transaction: resolved } : resolved;
492
+ }
493
+
494
+ /** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
495
+ function isPlainOptions(value: unknown): boolean {
496
+ return (
497
+ typeof value === 'object' &&
498
+ value !== null &&
499
+ !Array.isArray(value) &&
500
+ !(value instanceof URLSearchParams) &&
501
+ (value as any).id === undefined
502
+ );
503
+ }
504
+
383
505
  export function makeTable(options) {
384
506
  const {
385
507
  primaryKey,
@@ -398,6 +520,7 @@ export function makeTable(options) {
398
520
  description,
399
521
  hidden,
400
522
  cacheControl,
523
+ isBranch,
401
524
  } = options;
402
525
  let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
403
526
  evictionMs ??= 0;
@@ -410,6 +533,7 @@ export function makeTable(options) {
410
533
  if (!properties) properties = projectAttributesToProperties(attributes);
411
534
  const updateRecord = recordUpdater(primaryStore, tableId, auditStore);
412
535
  let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
536
+ let warnedFutureSourceVersion = false; // likewise (see getFromSource)
413
537
  let sourceLoad: any; // if a source has a load function (replicator), record it here
414
538
  let hasSourceGet: any;
415
539
  let primaryKeyAttribute: Attribute | undefined;
@@ -441,6 +565,10 @@ export function makeTable(options) {
441
565
  let cleanupPriority = 0;
442
566
  let lastCleanupInterval: number;
443
567
  let cleanupTimer: NodeJS.Timeout;
568
+ let recordExpirationInterval: NodeJS.Timeout;
569
+ // a reclamation pass awaits a scheduled cleanup, which only settles from its timer
570
+ const pendingCleanupResolvers = new Set<() => void>();
571
+ let disposed = false;
444
572
  // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
445
573
  let expirationScanScheduled = false;
446
574
  // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
@@ -489,9 +617,10 @@ export function makeTable(options) {
489
617
  const MAX_PREFETCH_SEQUENCE = 10;
490
618
  const MAX_PREFETCH_BUNDLE = 6;
491
619
  if (audit) addDeleteRemoval();
492
- onStorageReclamation(primaryStore.path, (priority: number) => {
620
+ const reclamationHandler = (priority: number) => {
493
621
  if (hasSourceGet) return scheduleCleanup(priority);
494
- });
622
+ };
623
+ onStorageReclamation(primaryStore.path, reclamationHandler);
495
624
 
496
625
  class Updatable extends GenericTrackedObject implements RecordObject {
497
626
  declare set: (property: string, value: any) => void;
@@ -582,13 +711,67 @@ export function makeTable(options) {
582
711
  },
583
712
  });
584
713
  }
714
+ function resolveAuditHead(
715
+ id: Id,
716
+ version: number | undefined,
717
+ nodeId: number | undefined,
718
+ refs?: Array<{ version: number; nodeId: number }>
719
+ ) {
720
+ if (!refs?.length) return { txnLogKey: version, nodeId };
721
+ const visited = new Set<string>();
722
+ function findHead(candidateRefs?: Array<{ version: number; nodeId: number }>) {
723
+ if (!candidateRefs) return;
724
+ const pending: Array<{ version: number; nodeId: number }> = candidateRefs.slice().reverse();
725
+ while (pending.length > 0) {
726
+ const ref = pending.pop()!;
727
+ const identity = `${ref.nodeId ?? 0}:${ref.version}`;
728
+ if (visited.has(identity)) continue;
729
+ visited.add(identity);
730
+ const entry = auditStore.getSync(ref.version, tableId, id, ref.nodeId);
731
+ if (!entry) continue;
732
+ if (entry.version === version && (nodeId == null || (entry.nodeId ?? 0) === nodeId))
733
+ return { txnLogKey: ref.version, nodeId: ref.nodeId };
734
+ const previousRefs = entry.previousAdditionalAuditRefs;
735
+ if (previousRefs) {
736
+ for (let index = previousRefs.length - 1; index >= 0; index--) pending.push(previousRefs[index]);
737
+ }
738
+ }
739
+ }
740
+ const referencedHead = findHead(refs);
741
+ if (referencedHead) return referencedHead;
742
+ if (version != null) {
743
+ const directHead = auditStore.getSync(version, tableId, id, nodeId);
744
+ if (directHead?.version === version && (nodeId == null || (directHead.nodeId ?? 0) === nodeId))
745
+ return { txnLogKey: version, nodeId };
746
+ }
747
+ return { txnLogKey: version, nodeId };
748
+ }
585
749
  class TableResource<Record extends object = any> extends Resource<Record> {
586
750
  #record: any; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
587
751
  #changes: any; // the changes to the record that have been made (should not be modified directly)
588
752
  #version?: number; // version of the record
589
753
  #entry?: Entry; // the entry from the database
590
754
  #savingOperation?: any; // operation for the record is currently being saved
755
+ #lockHandle?: RecordLockHandle; // the record lock acquired by lock() — scoped or hold
756
+ #lockWritable?: boolean; // set by #reloadLocked to let save() stage lock-writable updates
591
757
  declare getProperty: (name: string) => any;
758
+
759
+ /**
760
+ * Shared guard: if this instance is lock-writable but the handle is gone (expired or
761
+ * released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
762
+ * in addition to the save() path. Every lock-writable instance carries its own handle in
763
+ * #lockHandle (scoped and hold alike), so we never need to search the registry here.
764
+ */
765
+ #assertLiveHandle(id: Id): void {
766
+ if (!this.#lockWritable) return;
767
+ const handle = this.#lockHandle!;
768
+ // Off-key writes through the same resource instance are ordinary; only guard the
769
+ // exact key the lock was acquired for.
770
+ if (handle.keyId !== writeKeyId(id)) return;
771
+ if (handle.expired || handle.released) {
772
+ throw lockNotHeldError(handle);
773
+ }
774
+ }
592
775
  // #section: static-config
593
776
  static name = tableName; // for display/debugging purposes
594
777
  static primaryStore = primaryStore;
@@ -704,6 +887,9 @@ export function makeTable(options) {
704
887
  ensureLoaded: false,
705
888
  nodeId: event.nodeId,
706
889
  viaNodeId: event.viaNodeId,
890
+ // the origin's record version, stored as-is so every replica holds the version the
891
+ // origin holds; the transaction's own timestamp stays the origin's log key
892
+ version: event.version,
707
893
  // use per-event expiresAt: batched txn context only holds the first event's expiration
708
894
  expiresAt: event.expiresAt,
709
895
  // bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log
@@ -940,7 +1126,9 @@ export function makeTable(options) {
940
1126
  continue;
941
1127
  }
942
1128
  }
943
- // use the version as the transaction timestamp
1129
+ // A source that reports no log position of its own (no `timestamp`) has only one clock,
1130
+ // so its record version doubles as the apply transaction's timestamp. A replication
1131
+ // receiver always sets `timestamp` from the origin's log key and never reaches this.
944
1132
  if (!event.timestamp && event.version) event.timestamp = event.version;
945
1133
  const commitResolution = transaction(event, () => {
946
1134
  if (event.type === 'transaction') {
@@ -1414,7 +1602,26 @@ export function makeTable(options) {
1414
1602
  return coerceType(id, primaryKeyAttribute);
1415
1603
  }
1416
1604
 
1605
+ /**
1606
+ * A branch's Table classes deliberately carry the BASE's logical database name so an
1607
+ * application's schema and code resolve unchanged (harper#643). That makes every schema
1608
+ * mutation resolve against the global catalog — a `dropTable()` through a branch would delete
1609
+ * the live base table. Reads and writes are per-branch and unaffected; DDL is refused until a
1610
+ * branch owns a schema identity of its own.
1611
+ */
1612
+ static assertSchemaMutable(operation: string) {
1613
+ if (!isBranch) return;
1614
+ const error: any = new Error(
1615
+ `Cannot ${operation} through a branched database: '${tableName}' resolves to the schema of base ` +
1616
+ `database '${databaseName}', so the change would apply to the base rather than the branch`
1617
+ );
1618
+ error.statusCode = 400;
1619
+ throw error;
1620
+ }
1621
+
1417
1622
  static async dropTable() {
1623
+ TableResource.assertSchemaMutable('drop a table');
1624
+ const rootStore = primaryStore.rootStore;
1418
1625
  if (databaseName === databasePath) {
1419
1626
  // Persist a drop tombstone on the primary catalog entry BEFORE any
1420
1627
  // destructive work. If the process dies or a column family drop fails
@@ -1423,8 +1630,9 @@ export function makeTable(options) {
1423
1630
  // completeInterruptedDrop in databases.ts instead of resurrecting
1424
1631
  // the table.
1425
1632
  const primaryCatalogKey = TableResource.tableName + '/';
1426
- const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey);
1427
- if (primaryMeta && !primaryMeta.dropping) {
1633
+ const writeTombstone = () => {
1634
+ const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey);
1635
+ if (!primaryMeta || primaryMeta.dropping) return;
1428
1636
  primaryMeta.dropping = true;
1429
1637
  // Stamps this drop's identity so the interrupted-drop retry budget in
1430
1638
  // databases.ts can be scoped to THIS drop rather than the table name: a
@@ -1434,11 +1642,28 @@ export function makeTable(options) {
1434
1642
  // the budget by generation instead makes the new drop's tombstone carry
1435
1643
  // its own fresh key regardless of what any worker last observed.
1436
1644
  primaryMeta.dropGeneration = randomUUID();
1437
- // put is rebound to putSync on RocksDB stores; on LMDB it returns
1438
- // a promise, so await it to make the tombstone durable before the
1439
- // destructive work below
1440
- const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta);
1441
- if (tombstoneWrite?.then) await tombstoneWrite;
1645
+ return (dbisDb as any).put(primaryCatalogKey, primaryMeta);
1646
+ };
1647
+ if (rootStore instanceof RocksDatabase) {
1648
+ // withUpdateAttributesLock's locked section cannot be held across an await, so a durable
1649
+ // tombstone depends on put being rebound to putSync for RocksDB primary stores (see
1650
+ // createOpenDBIObject). Check that BEFORE writing anything: a tombstone left behind by a
1651
+ // refused drop would delete the table on the next load.
1652
+ if ((dbisDb as any).put !== (dbisDb as any).putSync)
1653
+ throw new Error(
1654
+ `Cannot drop ${databaseName}.${TableResource.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`
1655
+ );
1656
+ withUpdateAttributesLock(
1657
+ rootStore,
1658
+ `drop table '${databaseName}.${TableResource.tableName}'`,
1659
+ writeTombstone
1660
+ );
1661
+ } else {
1662
+ let tombstoneWrite;
1663
+ rootStore.transactionSync(() => {
1664
+ tombstoneWrite = writeTombstone();
1665
+ });
1666
+ if (typeof tombstoneWrite?.then === 'function') await tombstoneWrite;
1442
1667
  }
1443
1668
  }
1444
1669
  // A get() against a sourcedFrom table resolves to its caller before the resolved
@@ -1518,19 +1743,16 @@ export function makeTable(options) {
1518
1743
  dbisDb.remove(TableResource.tableName + '/');
1519
1744
  return true;
1520
1745
  };
1521
- const rootStore = primaryStore.rootStore;
1522
1746
  if (rootStore instanceof RocksDatabase) {
1523
1747
  // Serialize the drops + catalog removal against a concurrent
1524
1748
  // same-name create (and completeInterruptedDrop) under the database's
1525
1749
  // 'update-attributes' exclusive lock - the same lock the create path
1526
- // holds. It is a synchronous spin lock that blocks the event loop, so
1750
+ // holds. It is a synchronous lock wait that blocks the event loop, so
1527
1751
  // the locked section MUST stay synchronous: drop with dropSync (as
1528
1752
  // completeInterruptedDrop does), never an awaited drop(), or a
1529
- // concurrent create's spin would deadlock waiting on a drop that the
1530
- // blocked event loop can never resolve.
1531
- while (!rootStore.tryLock('update-attributes')) {}
1532
- let removed = false;
1533
- try {
1753
+ // concurrent create's wait would be stuck on a drop that the blocked
1754
+ // event loop can never resolve, burning its full deadline before failing.
1755
+ const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
1534
1756
  for (const attribute of attributes) {
1535
1757
  const index = indices[attribute.name];
1536
1758
  if (index)
@@ -1545,10 +1767,8 @@ export function makeTable(options) {
1545
1767
  } catch (error) {
1546
1768
  ignoreAlreadyDropped(error);
1547
1769
  }
1548
- removed = removeTombstonedCatalog();
1549
- } finally {
1550
- rootStore.unlock('update-attributes');
1551
- }
1770
+ return removeTombstonedCatalog();
1771
+ });
1552
1772
  if (removed) await dbisDb.committed;
1553
1773
  } else {
1554
1774
  // LMDB: no shared column-family double-drop, and its engine lock is
@@ -1564,7 +1784,12 @@ export function makeTable(options) {
1564
1784
  if (removeTombstonedCatalog()) await dbisDb.committed;
1565
1785
  }
1566
1786
  } else {
1567
- // legacy table per database
1787
+ // legacy table per database. The store to retire is this table's own audit store: nothing
1788
+ // assigns `primaryStore.auditStore` — openAuditStore() assigns `rootStore.auditStore`, and
1789
+ // this is the reference makeTable() was handed. Awaited so a pass suspended mid-removal has
1790
+ // released the primary DBI before it is closed and unlinked.
1791
+ await auditStore?.stopAuditCleanup?.();
1792
+ removeStorageReclamation(primaryStore.path);
1568
1793
  await primaryStore.close();
1569
1794
  fs.unlinkSync(primaryStore.path);
1570
1795
  }
@@ -1884,13 +2109,71 @@ export function makeTable(options) {
1884
2109
  * Save any changes into this instance to the current transaction
1885
2110
  */
1886
2111
  save() {
1887
- if (this.#savingOperation) {
1888
- try {
1889
- return this.#saveOperation(this.#savingOperation);
1890
- } finally {
2112
+ this.#assertLiveHandle(this.getId()); // a write through a released or expired lock never lands
2113
+ const operation = this.#savingOperation;
2114
+ if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
2115
+ // A held lock's record stages its update here rather than at lock() time: it is often
2116
+ // written after the acquiring transaction has already completed, which would have
2117
+ // dropped an update staged then. Nothing set means nothing to stage — a held-but-untouched
2118
+ // id stays untouched. Scoped locks do not take this branch: #reloadLocked stages their
2119
+ // TransactionWrite at lock() time (exactly like update()), so #savingOperation is always
2120
+ // set for a live scoped lock and the ordinary path below applies.
2121
+ // Verify the hold is still alive: if #lockWritable is set but the handle expired or was
2122
+ // released between lock acquisition and this save(), throw 409 rather than silently
2123
+ // committing stale data. Every lock-writable instance carries its own handle.
2124
+ const saveHandle = this.#lockHandle!;
2125
+ if (saveHandle.expired || saveHandle.released) {
2126
+ throw lockNotHeldError(saveHandle);
2127
+ }
2128
+ const changes = this.#changes;
2129
+ if (changes && Object.keys(changes).length > 0) {
2130
+ this.#savingOperation = null;
2131
+ return when(this._writeUpdate(this.getId(), changes, false), () => {
2132
+ const op = this.#savingOperation;
2133
+ if (op?.dropped) {
2134
+ this.#changes = undefined;
2135
+ return;
2136
+ }
2137
+ // Clear #savingOperation so the next sequential save() enters the lock-writable
2138
+ // path and creates a fresh write (otherwise a non-null #savingOperation makes
2139
+ // save() take the #saveOperation branch with an already-committed write, which
2140
+ // is a no-op, silently dropping the new change).
2141
+ // op.innerCommit is the real native-transaction commit Promise set on the
2142
+ // immediateCommit path in DatabaseTransaction.save(); await it to ensure
2143
+ // durability before resolving to the caller.
2144
+ if (op?.saved) {
2145
+ this.#savingOperation = null;
2146
+ return op?.innerCommit;
2147
+ }
2148
+ // op.saved = false means addWrite deferred the save; #saveOperation commits it
2149
+ // synchronously but ImmediateTransaction.save() returns undefined while the
2150
+ // inner rocksdb commit is still pending — return innerCommit so the caller
2151
+ // actually waits for durability.
2152
+ return when(this.save(), () => op?.innerCommit);
2153
+ });
2154
+ }
2155
+ // No changes: nothing to stage. A dropped operation (detached at a scoped→hold
2156
+ // upgrade — see detachScopedUpgradeWrite) must not fall through to the ordinary
2157
+ // #saveOperation path below with its now-detached reference.
2158
+ if (!operation || operation.dropped) {
1891
2159
  this.#savingOperation = null;
2160
+ return;
1892
2161
  }
1893
2162
  }
2163
+ if (this.#savingOperation) {
2164
+ const operation = this.#savingOperation;
2165
+ this.#savingOperation = null;
2166
+ // A write that lands via a nested immediateCommit (e.g. a second sequential save() on
2167
+ // the same ImmediateTransaction context, once the first has already closed it) sets
2168
+ // operation.innerCommit to the real native-commit promise, but the commit() sweep loop
2169
+ // that triggers it discards its own return value — #saveOperation()'s result can
2170
+ // resolve before that native commit actually settles. Chain on innerCommit (as the
2171
+ // lock-writable hold branch above already does) so callers awaiting save() see the
2172
+ // write durably land, not just the outer (possibly premature) resolution.
2173
+ const result = this.#saveOperation(operation);
2174
+ const innerCommit = operation.innerCommit;
2175
+ return innerCommit ? when(innerCommit, () => result) : result;
2176
+ }
1894
2177
  }
1895
2178
  #saveOperation(operation: any) {
1896
2179
  const transaction = txnForContext(this.getContext());
@@ -1968,6 +2251,7 @@ export function makeTable(options) {
1968
2251
  });
1969
2252
  }
1970
2253
  _writeInvalidate(id: Id, partialRecord?: any, options?: any) {
2254
+ this.#assertLiveHandle(id);
1971
2255
  const context = this.getContext();
1972
2256
  checkValidId(id);
1973
2257
  const transaction = txnForContext(this.getContext());
@@ -1976,9 +2260,13 @@ export function makeTable(options) {
1976
2260
  store: primaryStore,
1977
2261
  invalidated: true,
1978
2262
  entry: this.#entry,
2263
+ recordVersion: options?.version,
2264
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
1979
2265
  commit: (txnTime, existingEntry, _retry, transaction: any) => {
2266
+ const txnLogKey =
2267
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
1980
2268
  write.skipped = false; // reset on each retry; cleanup happens after commit if still true
1981
- if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) {
2269
+ if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
1982
2270
  write.skipped = true;
1983
2271
  return;
1984
2272
  }
@@ -2005,9 +2293,15 @@ export function makeTable(options) {
2005
2293
  viaNodeId: options?.viaNodeId,
2006
2294
  transaction,
2007
2295
  tableToTrack: tableName,
2296
+ recordVersion: txnTime,
2297
+ additionalAuditRefs:
2298
+ isRocksDB && audit && txnLogKey !== txnTime
2299
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
2300
+ : undefined,
2008
2301
  },
2009
2302
  'invalidate'
2010
2303
  );
2304
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
2011
2305
  // TODO: recordDeletion?
2012
2306
  },
2013
2307
  };
@@ -2015,20 +2309,25 @@ export function makeTable(options) {
2015
2309
  transaction.addWrite(write);
2016
2310
  }
2017
2311
  _writeRelocate(id: Id, options: any) {
2312
+ this.#assertLiveHandle(id);
2018
2313
  const context = this.getContext();
2019
2314
  checkValidId(id);
2020
2315
  const transaction = txnForContext(this.getContext());
2021
- transaction.addWrite({
2316
+ const write: any = {
2022
2317
  key: id,
2023
2318
  store: primaryStore,
2024
2319
  invalidated: true,
2025
2320
  entry: this.#entry,
2321
+ recordVersion: options?.version,
2322
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
2026
2323
  before:
2027
2324
  (this.constructor as any).source?.relocate && !(context as any)?.source
2028
2325
  ? (this.constructor as any).source.relocate.bind((this.constructor as any).source, id, undefined, context)
2029
2326
  : undefined,
2030
2327
  commit: (txnTime, existingEntry, _retry, transaction: any) => {
2031
- if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return;
2328
+ const txnLogKey =
2329
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2330
+ if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) return;
2032
2331
  const residency = TableResource.getResidencyRecord(options.residencyId);
2033
2332
  let metadata = 0;
2034
2333
  let newRecord = null;
@@ -2060,13 +2359,20 @@ export function makeTable(options) {
2060
2359
  viaNodeId: options?.viaNodeId,
2061
2360
  expiresAt: options.expiresAt,
2062
2361
  transaction,
2362
+ recordVersion: txnTime,
2363
+ additionalAuditRefs:
2364
+ isRocksDB && audit && txnLogKey !== txnTime
2365
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
2366
+ : undefined,
2063
2367
  },
2064
2368
  'relocate',
2065
2369
  false,
2066
2370
  null
2067
2371
  );
2372
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
2068
2373
  },
2069
- });
2374
+ };
2375
+ transaction.addWrite(write);
2070
2376
  }
2071
2377
 
2072
2378
  /**
@@ -2182,10 +2488,260 @@ export function makeTable(options) {
2182
2488
  }
2183
2489
  }
2184
2490
  /**
2185
- * This is intended to acquire a lock on a record from the whole cluster.
2491
+ * Static entry point: `Table.lock(id, options?, context?)` — creates an instance in the given,
2492
+ * ambient, or a fresh context and delegates to the instance lock(). This shadows Resource.static
2493
+ * lock so that both callers share the same transaction link (required for cross-instance upgrade
2494
+ * detection). lock() is an in-process API with no authorization hook of its own; it is not
2495
+ * protocol-dispatched, so no allowUpdate/allowCreate check runs on acquisition.
2496
+ *
2497
+ * Dropping the trailing `context` leaks the key: the bare `{}` fallback is an
2498
+ * ImmediateTransaction, which releases no record locks.
2186
2499
  */
2187
- lock() {
2188
- throw new Error('Not yet implemented');
2500
+ static async lock(
2501
+ target?: RequestTargetOrId | RecordLockOptions,
2502
+ options?: RecordLockOptions,
2503
+ context?: any
2504
+ ): Promise<any> {
2505
+ if (!isRocksDB) throw new ClientError('Record locks are not supported on LMDB', 501);
2506
+ if (options === undefined && isPlainOptions(target)) {
2507
+ options = target as RecordLockOptions;
2508
+ target = undefined;
2509
+ }
2510
+ const id = target != null ? requestTargetToId(target as RequestTargetOrId) : null;
2511
+ const resolvedContext: any = contextArgument(context) ?? contextStorage.getStore() ?? {};
2512
+ const resource = new TableResource(id, resolvedContext);
2513
+ return resource.lock(target, options);
2514
+ }
2515
+ /**
2516
+ * Acquire an exclusive lock on this record (or on `target`'s) and return it ready for updates
2517
+ * (harper#483, Phase 0: exclusive across every worker thread of this node). The lock is held
2518
+ * in process memory only — no durable writes. Phase 0 contract: lock() is mutually exclusive
2519
+ * with other lock() calls on the same key; plain writes (put/patch/delete/create) are never
2520
+ * gated or blocked. The generation expires after `lease` if it is never released.
2521
+ *
2522
+ * Transaction-scoped (default): write through the returned record (or the table's static verbs
2523
+ * in the same transaction), and the commit or abort releases it. `{ hold: true }`: the lock
2524
+ * outlives the transaction; write through the returned record and release with `unlock()`, or
2525
+ * let the lease expire.
2526
+ */
2527
+ // async so option/id validation rejects rather than throwing past a caller's `.catch()`; the
2528
+ // body still runs to completion synchronously, which is what keeps concurrent lock() calls
2529
+ // on one key coalescing instead of racing to tryLock.
2530
+ async lock(target?: RequestTargetOrId | RecordLockOptions, options?: RecordLockOptions): Promise<any> {
2531
+ if (!isRocksDB) throw new ClientError('Record locks are not supported on LMDB', 501);
2532
+ if (options === undefined && isPlainOptions(target)) {
2533
+ options = target as RecordLockOptions;
2534
+ target = undefined;
2535
+ }
2536
+ const id = target != null ? requestTargetToId(target as RequestTargetOrId) : this.getId();
2537
+ checkValidId(id);
2538
+ const resolved = resolveLockOptions(options);
2539
+ const context = this.getContext();
2540
+ const link = txnForContext(context);
2541
+ const keyId = writeKeyId(id);
2542
+ const held = this.#lockHandle;
2543
+ if (held && !held.released && !held.expired && held.keyId === keyId) {
2544
+ // Re-entrant: upgrade to hold if requested, then preserve staged changes.
2545
+ if (resolved.hold && !held.hold) {
2546
+ held.upgradeToHold(resolved.lease);
2547
+ // The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
2548
+ // staging is deferred and explicit-save-only, so an unsaved scoped write left
2549
+ // dangling here would otherwise auto-commit at the transaction sweep and clobber
2550
+ // whatever the hold write lands. detachScopedUpgradeWrite marks it .dropped so a
2551
+ // later save() on this instance falls through to the hold branch instead of the
2552
+ // dead #savingOperation reference.
2553
+ detachScopedUpgradeWrite(link, keyId, held);
2554
+ }
2555
+ return Promise.resolve(this.#reloadLocked(id, undefined, true));
2556
+ }
2557
+ const scoped = link.recordLockFor(primaryStore, keyId);
2558
+ if (scoped && !scoped.released && !scoped.expired) {
2559
+ if (resolved.hold && !scoped.hold) {
2560
+ // Upgrade scoped → hold: flip the existing handle object to hold mode so every
2561
+ // instance that already references this handle stays valid. Retiring and creating a
2562
+ // new handle would invalidate those other references (their save() would then throw
2563
+ // 409 against a released handle). The native key stays locked throughout.
2564
+ scoped.upgradeToHold(resolved.lease);
2565
+ detachScopedUpgradeWrite(link, keyId, scoped);
2566
+ return Promise.resolve(this.#reloadLocked(id, scoped, true));
2567
+ }
2568
+ // Already held with the same type: re-entrant return. Preserve any staged changes.
2569
+ return Promise.resolve(this.#reloadLocked(id, scoped, true));
2570
+ }
2571
+ const key = lockAttemptKey(tableId, id);
2572
+ // Coalesce concurrent lock() calls for the same key inside one link so they don't
2573
+ // self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
2574
+ // reach tryLock before either registers, making the second park against the first.
2575
+ const pending = link.pendingLockFor(primaryStore, keyId);
2576
+ if (pending) {
2577
+ // Wait for the in-flight acquisition, then take the re-entrant path as if
2578
+ // recordLockFor had found it. If the first attempt timed out, re-enter so
2579
+ // the second caller gets its own timeout.
2580
+ // The follower waits on the leader's acquisition, but only for its own timeout.
2581
+ let followerTimer: ReturnType<typeof setTimeout> | undefined;
2582
+ const followerTimedOut = Symbol('follower timeout');
2583
+ const followerStart = Date.now();
2584
+ const followerDeadline = new Promise<never>((_, reject) => {
2585
+ followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
2586
+ });
2587
+ // Try again on this caller's own terms with the budget it has left.
2588
+ const retryOnRemainingBudget = () => {
2589
+ // The enclosing transaction ended while we were parked. A retry re-resolves the
2590
+ // context, which no longer points at this link, so the handle it acquired would be
2591
+ // registered on a fresh transaction that no commit or abort ever releases — the
2592
+ // same abandonment the leader's own post-acquisition guard below rejects.
2593
+ if (link.open === TRANSACTION_STATE.CLOSED && !link.saveCommits)
2594
+ throw new ServerError('Transaction was closed while waiting for a record lock', 500);
2595
+ const remaining = resolved.timeout - (Date.now() - followerStart);
2596
+ if (remaining <= 0) throw new ClientError(`Record is locked and was not released in time`, 423);
2597
+ return this.lock(target, { ...resolved, timeout: remaining }) as Promise<any>;
2598
+ };
2599
+ return Promise.race([pending, followerDeadline]).then(
2600
+ () => {
2601
+ clearTimeout(followerTimer);
2602
+ const acquired = link.recordLockFor(primaryStore, keyId);
2603
+ if (acquired && !acquired.released && !acquired.expired) {
2604
+ if (resolved.hold && !acquired.hold) {
2605
+ detachScopedUpgradeWrite(link, keyId, acquired);
2606
+ acquired.upgradeToHold(resolved.lease);
2607
+ }
2608
+ return this.#reloadLocked(id, acquired, true);
2609
+ }
2610
+ return retryOnRemainingBudget();
2611
+ },
2612
+ (error) => {
2613
+ clearTimeout(followerTimer);
2614
+ if (error === followerTimedOut) throw new ClientError(`Record is locked and was not released in time`, 423);
2615
+ return retryOnRemainingBudget();
2616
+ }
2617
+ );
2618
+ }
2619
+ const pendingPromise = acquireRecordKey(
2620
+ link,
2621
+ primaryStore,
2622
+ key,
2623
+ keyId,
2624
+ resolved.timeout,
2625
+ resolved.lease,
2626
+ resolved.hold
2627
+ );
2628
+ link.registerPendingLock(primaryStore, keyId, pendingPromise);
2629
+ return pendingPromise.then(
2630
+ (handle) => {
2631
+ link.unregisterPendingLock(primaryStore, keyId);
2632
+ if (link.open === TRANSACTION_STATE.CLOSED && !link.saveCommits) {
2633
+ // The transaction was aborted while this call waited; nothing would ever release the handle.
2634
+ handle.release();
2635
+ throw new ServerError('Transaction was closed while waiting for a record lock', 500);
2636
+ }
2637
+ link.registerRecordLock(handle);
2638
+ if (link.saveCommits && (context as any)?.timestamp) handle.noteCandidateFloor((context as any).timestamp);
2639
+ if (link.open === TRANSACTION_STATE.OPEN && !link.saveCommits) {
2640
+ // Explicit transaction() (not ImmediateTransaction): pin the clock to
2641
+ // acquiredAt when no writes have been staged yet. When writes already
2642
+ // exist, leave the clock alone (ordering is best-effort; write held records
2643
+ // in their own transaction for the guarantee). ImmediateTransaction is
2644
+ // excluded (saveCommits=true) — its clock is never pinned in lock();
2645
+ // each save() stamps from the handle's committed version floor instead.
2646
+ if (link.writes.length === 0 && !link.timestamp) {
2647
+ link.timestamp = handle.acquiredAt;
2648
+ }
2649
+ if (!resolved.hold && link.transaction) {
2650
+ // Scoped lock: the read snapshot may predate the lock; drop it so the
2651
+ // scope reads what it locked. Hold locks use acquiredAt directly and
2652
+ // do not update the read snapshot.
2653
+ // The timestamp guard matches DatabaseTransaction's own setTimestamp calls: a
2654
+ // deferred update() write leaves the clock at 0, which rocksdb-js rejects.
2655
+ if (link.writes.length === 0 && link.readTxnsUsed <= 1) {
2656
+ link.releaseReadTxn();
2657
+ link.snapshotFree = true;
2658
+ } else if (link.timestamp) link.transaction.setTimestamp(link.timestamp);
2659
+ }
2660
+ }
2661
+ // ImmediateTransaction: no clock pinning in lock(); save() stamps each write
2662
+ // from the committed handle floor for both scoped and hold handles.
2663
+ return this.#reloadLocked(id, handle);
2664
+ },
2665
+ (err) => {
2666
+ link.unregisterPendingLock(primaryStore, keyId);
2667
+ throw err;
2668
+ }
2669
+ );
2670
+ }
2671
+ #reloadLocked(id: Id, holdHandle?: RecordLockHandle | null, preserveChanges = false) {
2672
+ // For freshness, read the committed entry (snapshot-free) so a hold lock sees concurrent
2673
+ // committed writes rather than a stale snapshot. A write earlier in THIS explicit
2674
+ // transaction has not landed in that committed entry yet (harper#1968: Harper defers an
2675
+ // explicit transaction's writes until the writing call actually runs them), so pull the
2676
+ // current value the same way a chained write picks up its basis (priorStagedWrite): the
2677
+ // record comes from the prior staged write, the rest of the entry (version, audit chain,
2678
+ // blob metadata) stays the pre-transaction one.
2679
+ const link = txnForContext(this.getContext());
2680
+ let entryForReload: any = primaryStore.getEntry(id);
2681
+ if (link.open === TRANSACTION_STATE.OPEN) {
2682
+ const keyId = writeKeyId(id);
2683
+ const tailWrite = link.writesByKey?.get(primaryStore)?.get(keyId);
2684
+ const priorStaged =
2685
+ tailWrite && (tailWrite.stagedEntry !== undefined ? tailWrite : priorStagedWrite(tailWrite));
2686
+ if (priorStaged?.stagedEntry !== undefined) {
2687
+ entryForReload = entryForReload
2688
+ ? { ...entryForReload, value: priorStaged.stagedEntry.value }
2689
+ : { value: priorStaged.stagedEntry.value };
2690
+ if (entryForReload.value && typeof entryForReload.value === 'object') {
2691
+ // Register the merged entry in entryMap so getUpdatedTime() works.
2692
+ entryMap.set(entryForReload.value, entryForReload);
2693
+ }
2694
+ }
2695
+ }
2696
+ if (writeKeyId(id) !== writeKeyId(this.getId())) {
2697
+ // lock(target) where target differs from this record: return a separate instance.
2698
+ const fresh = new (this.constructor as any)(id, this.getContext());
2699
+ TableResource._updateResource(fresh, entryForReload);
2700
+ if (holdHandle != null) {
2701
+ fresh.#lockHandle = holdHandle;
2702
+ // Do not clear this.#lockHandle: the original instance keeps its own lock on its
2703
+ // own id; the fresh instance owns the lock on the target id independently.
2704
+ }
2705
+ fresh.#lockWritable = true;
2706
+ // Scoped (not hold) stages exactly like update(): create the TransactionWrite now so
2707
+ // save() is the ordinary #savingOperation path. Hold keeps deferred staging (the
2708
+ // acquiring transaction may commit before the holder ever writes).
2709
+ if (!fresh.#lockHandle!.hold) fresh._writeUpdate(id, fresh.#changes, false);
2710
+ return fresh;
2711
+ }
2712
+ // Store the handle for both scoped and hold locks; undefined (re-entrant hold fast-path)
2713
+ // must not clear a handle already set.
2714
+ if (holdHandle != null) this.#lockHandle = holdHandle;
2715
+ TableResource._updateResource(this, entryForReload);
2716
+ // Preserve staged changes when upgrading the same instance from scoped to hold so that
2717
+ // set() calls made under the scoped lock survive the reload.
2718
+ if (!preserveChanges) this.#changes = undefined;
2719
+ this.#lockWritable = true;
2720
+ // Scoped (not hold): stage now, same as update() would. Skip if a write from an earlier
2721
+ // lock() cycle on this instance is still pending (re-entrant call before its save()).
2722
+ if (!this.#lockHandle!.hold && !this.#savingOperation) this._writeUpdate(id, this.#changes, false);
2723
+ return this;
2724
+ }
2725
+ /**
2726
+ * Release the lock this instance holds. Resolves true when this call cleared the native key lock.
2727
+ * Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
2728
+ * instance is no longer lock-writable; writes through it require a fresh lock.
2729
+ */
2730
+ unlock(): Promise<boolean> {
2731
+ // Always clear the local lock-writable state so subsequent writes on this instance are
2732
+ // ungated, regardless of whether the handle was already released.
2733
+ const handle = this.#lockHandle;
2734
+ this.#lockHandle = undefined;
2735
+ this.#lockWritable = false;
2736
+ if (!handle || handle.released) return Promise.resolve(false);
2737
+ const link = txnForContext(this.getContext());
2738
+ // A scoped lock staged its write at lock() time; released before commit, that write must not
2739
+ // run into the released-handle guard at the sweep.
2740
+ if (this.#savingOperation && !this.#savingOperation.saved && this.#savingOperation.lockHandle === handle)
2741
+ this.#savingOperation = null;
2742
+ detachScopedUpgradeWrite(link, writeKeyId(this.getId()), handle);
2743
+ link.unregisterRecordLock(handle);
2744
+ return Promise.resolve(handle.release());
2189
2745
  }
2190
2746
  static operation(operation, context) {
2191
2747
  operation.table ||= tableName;
@@ -2311,8 +2867,10 @@ export function makeTable(options) {
2311
2867
  // a notification that a write has already occurred in the canonical data source, we need to update our
2312
2868
  // local copy
2313
2869
  _writeUpdate(id: Id, recordUpdate: any, fullUpdate: boolean, options?: any) {
2870
+ this.#assertLiveHandle(id);
2314
2871
  const context = this.getContext();
2315
2872
  const transaction = txnForContext(context);
2873
+ const replaying = transaction.isReplay === true;
2316
2874
  checkValidId(id);
2317
2875
  if (fullUpdate && recordUpdate == null && options?.isNotification) {
2318
2876
  // A source/replication-applied put must carry the record; these applies skip record
@@ -2356,6 +2914,13 @@ export function makeTable(options) {
2356
2914
  nodeName: (context as any)?.nodeName,
2357
2915
  fullUpdate,
2358
2916
  deferSave: true,
2917
+ // the origin's record version on an applied write; absent for a locally-originated one
2918
+ recordVersion: options?.version,
2919
+ // Include the lock handle (if any) so the expired-handle guard in
2920
+ // DatabaseTransaction.save() can throw 409 when the lease has lapsed.
2921
+ // Only attach the hold handle when it covers exactly this key; off-key writes
2922
+ // are ordinary and must not carry an unrelated hold's handle.
2923
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
2359
2924
  validate: (txnTime, committedBy = transaction) => {
2360
2925
  if (!recordUpdate) recordUpdate = this.#changes;
2361
2926
  if (fullUpdate || (recordUpdate && hasChanges(this.#changes === recordUpdate ? this : recordUpdate))) {
@@ -2478,6 +3043,8 @@ export function makeTable(options) {
2478
3043
  this.#savingOperation = null;
2479
3044
  write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
2480
3045
  let omitLocalRecord = false;
3046
+ const txnLogKey =
3047
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
2481
3048
  // we use optimistic locking to only commit if the existing record state still holds true.
2482
3049
  // this is superior to using an async transaction since it doesn't require JS execution
2483
3050
  // during the write transaction.
@@ -2540,10 +3107,10 @@ export function makeTable(options) {
2540
3107
  if (
2541
3108
  existingEntry.additionalAuditRefs?.some(
2542
3109
  (ref) =>
2543
- ref.version === txnTime &&
3110
+ ref.version === txnLogKey &&
2544
3111
  precedesExistingVersion(
2545
3112
  txnTime,
2546
- { version: txnTime, localTime: txnTime, key: id, nodeId: ref.nodeId },
3113
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId },
2547
3114
  options?.nodeId
2548
3115
  ) === 0
2549
3116
  )
@@ -2574,10 +3141,10 @@ export function makeTable(options) {
2574
3141
  if (!oldestRetainedAuditTimeResolved) {
2575
3142
  oldestRetainedAuditTimeResolved = true;
2576
3143
  // getRange yields ascending by audit-log key, so the first entry is the oldest retained.
2577
- // Mirror replicationConnection's retention check and the cleanup key basis (localTime ??
2578
- // version). Fall back to the nominal time-based purge floor when the log is empty/unavailable.
3144
+ // Mirror replicationConnection's retention check and the cleanup key basis (`txnLogKey`).
3145
+ // Fall back to the nominal time-based purge floor when the log is empty/unavailable.
2579
3146
  for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
2580
- oldestRetainedAuditTime = entry.localTime ?? entry.version;
3147
+ oldestRetainedAuditTime = entry.txnLogKey;
2581
3148
  break;
2582
3149
  }
2583
3150
  oldestRetainedAuditTime ??= Date.now() - auditRetention;
@@ -2590,23 +3157,25 @@ export function makeTable(options) {
2590
3157
  // depth-cap block. This is the same keyed lookup that block performs, hoisted ahead of the walk.
2591
3158
  // It is what catches transitive/proxied re-deliveries: they arrive buried below the record head
2592
3159
  // (so replication's head-tie fast-skip can't see them) yet are exact duplicates. Keyed by nodeId,
2593
- // so it is correct across multiple source nodes. RocksDB-only: LMDB audit entries are keyed by
2594
- // local audit time, not version, so this version-keyed lookup doesn't apply there (LMDB keeps the
2595
- // exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
3160
+ // so it is correct across multiple source nodes. The lookup key is this write's LOG key, not its
3161
+ // record version — a replication apply commits under the origin's log key while storing the
3162
+ // origin's version, and only the log key addresses the entry (harper#2412).
3163
+ // RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
3164
+ // there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
2596
3165
  // simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
2597
3166
  // check above remains the read-your-writes guard. Never when this write staged in a prior
2598
3167
  // failed attempt: that attempt already appended this write's own audit entry, so the lookup
2599
3168
  // would find it and skip the write as "already applied" when the record was never committed.
2600
3169
  // A recommit of the same transaction survived that skip only because the old write batch
2601
3170
  // still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
2602
- if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnTime)) {
2603
- const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId);
3171
+ if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
3172
+ const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
2604
3173
  if (
2605
3174
  priorAudit &&
2606
- priorAudit.version === txnTime &&
3175
+ priorAudit.txnLogKey === txnLogKey &&
2607
3176
  precedesExistingVersion(
2608
3177
  txnTime,
2609
- { version: txnTime, localTime: txnTime, key: id, nodeId: priorAudit.nodeId },
3178
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: priorAudit.nodeId },
2610
3179
  options?.nodeId
2611
3180
  ) === 0
2612
3181
  ) {
@@ -2615,7 +3184,10 @@ export function makeTable(options) {
2615
3184
  }
2616
3185
  }
2617
3186
  // incremental CRDT updates are only available with audit logging on
2618
- let localTime = existingEntry.localTime;
3187
+ const initialAuditHead = isRocksDB
3188
+ ? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
3189
+ : { txnLogKey: existingEntry.localTime, nodeId: existingEntry.nodeId };
3190
+ let localTime = initialAuditHead.txnLogKey;
2619
3191
  let auditedVersion = existingEntry.version;
2620
3192
  logger.debug?.(
2621
3193
  'Applying CRDT update to record with id: ',
@@ -2628,22 +3200,42 @@ export function makeTable(options) {
2628
3200
  new Date(localTime)
2629
3201
  );
2630
3202
 
2631
- let nodeId = existingEntry.nodeId;
3203
+ let nodeId = initialAuditHead.nodeId;
2632
3204
  const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
2633
3205
  const auditRefsToVisit: Array<{ localTime: number; nodeId: number }> = existingEntry.additionalAuditRefs
2634
3206
  ? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
2635
3207
  : [];
2636
3208
 
2637
- // Collect any existing audit refs that should be preserved (those older than current transaction)
3209
+ // Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
2638
3210
  if (existingEntry.additionalAuditRefs) {
2639
3211
  for (const ref of existingEntry.additionalAuditRefs) {
2640
- if (ref.version <= txnTime) {
2641
- additionalAuditRefs.push(ref);
2642
- }
3212
+ additionalAuditRefs.push(ref);
2643
3213
  }
2644
3214
  }
2645
3215
  let addedAuditRef = false;
2646
3216
  let nextRef: { localTime: number; nodeId: number };
3217
+ const visitedAuditRefs = new Set<string>();
3218
+ const queuePreviousAuditRefs = (auditRecord) => {
3219
+ const previousRefs = auditRecord.previousAdditionalAuditRefs;
3220
+ if (previousRefs) {
3221
+ for (const ref of previousRefs) {
3222
+ auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
3223
+ logger.debug?.('Adding audit ref from audit record to visit queue', {
3224
+ version: ref.version,
3225
+ nodeId: ref.nodeId,
3226
+ });
3227
+ }
3228
+ }
3229
+ };
3230
+ const advanceToPreviousAudit = (auditRecord) => {
3231
+ const previousRefs = auditRecord.previousAdditionalAuditRefs;
3232
+ const previousHead =
3233
+ isRocksDB && previousRefs?.length
3234
+ ? resolveAuditHead(id, auditRecord.previousVersion, auditRecord.previousNodeId, previousRefs)
3235
+ : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
3236
+ localTime = previousHead.txnLogKey;
3237
+ nodeId = previousHead.nodeId;
3238
+ };
2647
3239
  let walkSteps = 0;
2648
3240
  let auditWalkCapped = false;
2649
3241
  // Early-out residual: as we walk the chain newest-first, fold each succeeding patch into a
@@ -2662,21 +3254,24 @@ export function makeTable(options) {
2662
3254
  // appended this write's own audit entry, so the lookup would match it while the record was
2663
3255
  // never committed (see the up-front keyed dedup above).
2664
3256
  const isReDeliveredDuplicate = () => {
2665
- if (stagedOwnAuditEntry) return false;
2666
- if (!dedupVersionCouldBeRetained(txnTime)) return false; // pre-retention version — skip the end-of-log scan (best-effort; see above)
2667
- const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId);
3257
+ if (replaying || stagedOwnAuditEntry) return false;
3258
+ if (!dedupVersionCouldBeRetained(txnLogKey)) return false; // pre-retention log key — skip the end-of-log scan (best-effort; see above)
3259
+ const duplicate = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
2668
3260
  return (
2669
3261
  duplicate &&
2670
- duplicate.version === txnTime &&
3262
+ duplicate.txnLogKey === txnLogKey &&
2671
3263
  precedesExistingVersion(
2672
3264
  txnTime,
2673
- { version: txnTime, localTime: txnTime, key: id, nodeId: duplicate.nodeId },
3265
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: duplicate.nodeId },
2674
3266
  options?.nodeId
2675
3267
  ) === 0
2676
3268
  );
2677
3269
  };
2678
3270
  do {
2679
3271
  while (localTime > txnTime || (auditedVersion >= txnTime && localTime > 0)) {
3272
+ const auditIdentity = `${nodeId ?? 0}:${localTime}`;
3273
+ if (visitedAuditRefs.has(auditIdentity)) break;
3274
+ visitedAuditRefs.add(auditIdentity);
2680
3275
  // Bound the walk only for RocksDB, where the OOM was observed (issue #1114): each step
2681
3276
  // is a transaction-log range scan + msgpackr decode, and the per-node logs can be huge.
2682
3277
  // LMDB audit entries are keyed by local audit time (not version), so the duplicate
@@ -2687,6 +3282,21 @@ export function makeTable(options) {
2687
3282
  }
2688
3283
  const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
2689
3284
  if (!auditRecord) break;
3285
+ queuePreviousAuditRefs(auditRecord);
3286
+ if (
3287
+ isRocksDB &&
3288
+ !replaying &&
3289
+ !stagedOwnAuditEntry &&
3290
+ localTime === txnLogKey &&
3291
+ precedesExistingVersion(
3292
+ txnTime,
3293
+ { version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId },
3294
+ options?.nodeId
3295
+ ) === 0
3296
+ ) {
3297
+ write.skipped = true;
3298
+ return;
3299
+ }
2690
3300
  auditedVersion = auditRecord.version;
2691
3301
  if (auditedVersion >= txnTime) {
2692
3302
  if (auditedVersion === txnTime) {
@@ -2696,17 +3306,26 @@ export function makeTable(options) {
2696
3306
  options?.nodeId
2697
3307
  );
2698
3308
  if (precedesExisting === 0) {
2699
- logger.debug?.(
2700
- 'The transaction time is equal to the existing version, treating as duplicate',
2701
- id
2702
- );
2703
- write.skipped = true;
2704
- return; // treat a tie as a duplicate and drop it
3309
+ if (isRocksDB && localTime !== txnLogKey) {
3310
+ // Same origin and record version, but a distinct write. Its per-origin log key
3311
+ // orders the otherwise non-unique record clock without comparing keys across origins.
3312
+ precedesExisting = txnLogKey > localTime ? 1 : -1;
3313
+ } else if (replaying || stagedOwnAuditEntry) {
3314
+ // The log entry being replayed (or staged by this write's failed attempt) is
3315
+ // the write itself, not proof that its primary-store mutation committed.
3316
+ precedesExisting = 1;
3317
+ } else {
3318
+ logger.debug?.(
3319
+ 'The transaction time and log key match the existing write, treating as duplicate',
3320
+ id
3321
+ );
3322
+ write.skipped = true;
3323
+ return;
3324
+ }
2705
3325
  }
2706
3326
  if (precedesExisting > 0) {
2707
3327
  // if the existing version is older, we can skip this update
2708
- localTime = auditRecord.previousVersion;
2709
- nodeId = auditRecord.previousNodeId;
3328
+ advanceToPreviousAudit(auditRecord);
2710
3329
  continue;
2711
3330
  }
2712
3331
  }
@@ -2743,24 +3362,16 @@ export function makeTable(options) {
2743
3362
  }
2744
3363
  if (!addedAuditRef && isRocksDB) {
2745
3364
  addedAuditRef = true;
2746
- // Add a reference to this older audit record if we had out-of-order writes
2747
- additionalAuditRefs.push({ version: txnTime, nodeId: options?.nodeId });
3365
+ // Add a reference to this older audit record if we had out-of-order writes. The stored
3366
+ // value is a LOG key, not a record version: every consumer follows it straight into
3367
+ // `auditStore.get` (see the `auditRefsToVisit` mapping above and below), and on an
3368
+ // applied write those two clocks differ.
3369
+ additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
2748
3370
  logger.debug?.('Adding additional audit ref for out-of-order write', {
2749
- version: txnTime,
3371
+ txnLogKey,
2750
3372
  nodeId: options?.nodeId,
2751
3373
  });
2752
3374
  }
2753
- // Collect any additional audit refs from this audit record to traverse other branches
2754
- if (auditRecord.previousAdditionalAuditRefs) {
2755
- for (const ref of auditRecord.previousAdditionalAuditRefs) {
2756
- auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
2757
- logger.debug?.('Adding audit ref from audit record to visit queue', {
2758
- version: ref.version,
2759
- nodeId: ref.nodeId,
2760
- });
2761
- }
2762
- }
2763
-
2764
3375
  // Every field of this write is overwritten by newer writes, and there is no alternate
2765
3376
  // audit branch left to scan, so it is fully superseded — the same outcome as walking to
2766
3377
  // the end and taking the `writeCommit(false)` escape below, reached without paying the rest
@@ -2779,8 +3390,7 @@ export function makeTable(options) {
2779
3390
  return writeCommit(false);
2780
3391
  }
2781
3392
 
2782
- localTime = auditRecord.previousVersion;
2783
- nodeId = auditRecord.previousNodeId;
3393
+ advanceToPreviousAudit(auditRecord);
2784
3394
  }
2785
3395
  // Check if we need to scan additional audit refs from this record
2786
3396
  if (auditWalkCapped) break;
@@ -2967,7 +3577,16 @@ export function makeTable(options) {
2967
3577
  );
2968
3578
  updateIndices(id, existingRecord, recordToStore, transaction && { transaction });
2969
3579
 
3580
+ // Preserve an addressable audit head when the record and log clocks diverge.
3581
+ if (isRocksDB && audit && !isCopyApply && txnLogKey !== txnTime) {
3582
+ const headIndex = additionalAuditRefs.findIndex(
3583
+ (ref) => ref.version === txnLogKey && (ref.nodeId ?? 0) === (options?.nodeId ?? 0)
3584
+ );
3585
+ if (headIndex > 0) additionalAuditRefs.unshift(additionalAuditRefs.splice(headIndex, 1)[0]);
3586
+ else if (headIndex < 0) additionalAuditRefs.unshift({ version: txnLogKey, nodeId: options?.nodeId });
3587
+ }
2970
3588
  writeCommit(true);
3589
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
2971
3590
  if (expiresAt >= 0) {
2972
3591
  scheduleCleanup(); // arm for replicated writes too, not just local-context writes
2973
3592
  // A runtime per-record expiresAt on a table with no table-level expiration/eviction, no expiresAt
@@ -3002,6 +3621,8 @@ export function makeTable(options) {
3002
3621
  user: (context as any)?.user,
3003
3622
  residencyId,
3004
3623
  expiresAt,
3624
+ recordVersion: txnTime,
3625
+ recordNodeId: precedesExisting < 0 ? existingEntry?.nodeId : options?.nodeId,
3005
3626
  nodeId: options?.nodeId,
3006
3627
  viaNodeId: options?.viaNodeId,
3007
3628
  originatingOperation: (context as any)?.originatingOperation,
@@ -3112,6 +3733,7 @@ export function makeTable(options) {
3112
3733
  return Boolean(this.#record);
3113
3734
  }
3114
3735
  _writeDelete(id: Id, options?: any) {
3736
+ this.#assertLiveHandle(id);
3115
3737
  const context = this.getContext();
3116
3738
  const transaction = txnForContext(context);
3117
3739
  checkValidId(id);
@@ -3123,6 +3745,8 @@ export function makeTable(options) {
3123
3745
  entry,
3124
3746
  chainsStagedState: true,
3125
3747
  nodeName: (context as any)?.nodeName,
3748
+ recordVersion: options?.version,
3749
+ lockHandle: this.#lockHandle && this.#lockHandle.keyId === writeKeyId(id) ? this.#lockHandle : undefined,
3126
3750
  before:
3127
3751
  (this.constructor as any).source?.delete && !(context as any)?.source
3128
3752
  ? (this.constructor as any).source.delete.bind((this.constructor as any).source, id, undefined, context)
@@ -3135,6 +3759,8 @@ export function makeTable(options) {
3135
3759
  const priorStagedOp = priorStagedWrite(write);
3136
3760
  const priorStaged = priorStagedOp?.stagedEntry;
3137
3761
  const existingRecord = priorStaged ? priorStaged.value : existingEntry?.value;
3762
+ const txnLogKey =
3763
+ isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
3138
3764
  if (retry) {
3139
3765
  if (context && existingEntry?.version > (context.lastModified || 0))
3140
3766
  context.lastModified = existingEntry.version;
@@ -3163,6 +3789,11 @@ export function makeTable(options) {
3163
3789
  viaNodeId: options?.viaNodeId,
3164
3790
  transaction,
3165
3791
  tableToTrack: tableName,
3792
+ recordVersion: txnTime,
3793
+ additionalAuditRefs:
3794
+ isRocksDB && audit && txnLogKey !== txnTime
3795
+ ? [{ version: txnLogKey, nodeId: options?.nodeId }]
3796
+ : undefined,
3166
3797
  },
3167
3798
  'delete'
3168
3799
  );
@@ -3172,6 +3803,7 @@ export function makeTable(options) {
3172
3803
  removeEntry(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
3173
3804
  }
3174
3805
  write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
3806
+ if (write.trackRecordVersion) write.recordVersionApplied = true;
3175
3807
  // the removal supersedes the nearest record an earlier write in this transaction stored
3176
3808
  // (older ones were already marked by their staged successors), so its saved blobs are
3177
3809
  // cleaned up post-commit unless its audit entry references them
@@ -3509,6 +4141,10 @@ export function makeTable(options) {
3509
4141
  }
3510
4142
  }
3511
4143
  const select = target.select;
4144
+ // Whether the caller supplied real filter conditions — read from the raw request, NOT the
4145
+ // planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
4146
+ // injected full-scan condition). Used to pick the count-estimate source below.
4147
+ const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
3512
4148
  if (conditions.length === 0) {
3513
4149
  conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
3514
4150
  }
@@ -3592,12 +4228,117 @@ export function makeTable(options) {
3592
4228
  readTxn,
3593
4229
  transformToRecord
3594
4230
  );
4231
+ const offset = target.offset || 0;
4232
+ const end = target.limit !== undefined ? offset + (target.limit as number) : undefined;
4233
+ // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
4234
+ // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
4235
+ // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
4236
+ // table estimate. Opt-in only — the default streaming path below is untouched.
4237
+ //
4238
+ // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
4239
+ // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
4240
+ // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
4241
+ // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
4242
+ // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
4243
+ // normal streaming path with no count. This bounds the offset too: without it a huge offset would
4244
+ // postpone the exact guardrail (which only engages past the page) until that offset was scanned.
4245
+ const pageLimit = target.limit as number;
4246
+ if (
4247
+ target.count &&
4248
+ Number.isInteger(pageLimit) &&
4249
+ pageLimit >= 0 &&
4250
+ pageLimit <= MAX_COUNT_PAGE &&
4251
+ Number.isInteger(offset) &&
4252
+ offset >= 0 &&
4253
+ offset + pageLimit <= MAX_EXACT_COUNT_SCAN
4254
+ ) {
4255
+ const wantExact = target.count === 'exact';
4256
+ const pageEnd = offset + pageLimit;
4257
+ const countStart = performance.now();
4258
+ // A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
4259
+ // chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
4260
+ // the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
4261
+ // different `count=exact` totals. Any query whose execution touches a custom index is affected: a
4262
+ // custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
4263
+ // filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
4264
+ // filter. Report the total as unavailable for those rather than advertising it as count=exact
4265
+ // (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
4266
+ // sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
4267
+ const touchesCustomIndex = (conds: any[]): boolean =>
4268
+ conds.some((c: any) => {
4269
+ if (!c) return false;
4270
+ if (c.conditions) return touchesCustomIndex(c.conditions);
4271
+ const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
4272
+ return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
4273
+ });
4274
+ const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
4275
+ return (async () => {
4276
+ const page: any = [];
4277
+ let scanned = 0;
4278
+ let exact = true;
4279
+ try {
4280
+ for await (const record of results) {
4281
+ if (scanned >= offset && scanned < pageEnd) page.push(record);
4282
+ scanned++;
4283
+ // A store whose async iterator settles synchronously (the common indexed-scan case) would
4284
+ // otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
4285
+ // for the whole count. Yield to the macrotask queue periodically so concurrent requests and
4286
+ // I/O still make progress during a large exact scan.
4287
+ if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0) await new Promise((resolve) => setImmediate(resolve));
4288
+ // The page window [offset, pageEnd) is always collected in full first — the guardrail
4289
+ // only ever abandons the running TOTAL, never truncates the page body.
4290
+ if (scanned >= pageEnd) {
4291
+ // `estimated` needs nothing past the page; an approximate (vector) exact total is going to
4292
+ // be reported unavailable anyway, so don't drain its tail for a number we won't publish.
4293
+ if (!wantExact || approximateResultSet) break;
4294
+ // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
4295
+ // large match set can't turn a bounded page fetch into an unbounded scan.
4296
+ if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) {
4297
+ exact = false;
4298
+ break;
4299
+ }
4300
+ }
4301
+ }
4302
+ } finally {
4303
+ // We own the iteration here (no results.onDone consumer), so release the read
4304
+ // transaction unconditionally — including when the drain throws — or the snapshot leaks.
4305
+ txn.doneReadTxn();
4306
+ }
4307
+ let total: number | null;
4308
+ if (wantExact) {
4309
+ // `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
4310
+ // an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
4311
+ total = exact && !approximateResultSet ? scanned : null;
4312
+ } else if (boundRowFilter || typeof target.vectorFilter === 'function') {
4313
+ // An opaque row/vector filter shapes the result but isn't reflected in the index/condition
4314
+ // estimate; guessing would both mislead and disclose cardinality the filter hides.
4315
+ total = null;
4316
+ } else if (!hasUserConditions) {
4317
+ total = estimatedEntryCount(primaryStore);
4318
+ } else {
4319
+ // Estimate from the real conditions only — drop the planner's synthetic `sort`
4320
+ // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
4321
+ const est = estimateCondition(TableResource)({
4322
+ conditions: conditions.filter((c: any) => c.comparator !== 'sort'),
4323
+ operator: operator ? String(operator).toLowerCase() : 'and',
4324
+ });
4325
+ total = isFinite(est) ? Math.round(est) : null;
4326
+ }
4327
+ // For an estimate, never report a total below the last row actually returned — keeps the
4328
+ // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
4329
+ // Exact totals are authoritative (and an empty page past the end must not be clamped up).
4330
+ if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
4331
+ total = offset + page.length;
4332
+ }
4333
+ page.recordCount = total;
4334
+ page.recordCountExact = wantExact && exact && !approximateResultSet;
4335
+ page.selectApplied = true;
4336
+ page.getColumns = getColumns;
4337
+ return page;
4338
+ })() as any;
4339
+ }
3595
4340
  // apply any offset/limit after all the sorting and filtering
3596
- if (target.offset || target.limit !== undefined)
3597
- results = results.slice(
3598
- target.offset,
3599
- target.limit !== undefined ? (target.offset || 0) + target.limit : undefined
3600
- );
4341
+ if (target.offset || target.limit !== undefined) results = results.slice(offset, end);
3601
4342
  results.onDone = () => {
3602
4343
  results.onDone = null; // ensure that it isn't called twice
3603
4344
  txn.doneReadTxn();
@@ -4056,6 +4797,10 @@ export function makeTable(options) {
4056
4797
  }
4057
4798
  if (!auditStore) throw new Error('Can not subscribe to a table without an audit log');
4058
4799
  if (!audit) {
4800
+ // Turning auditing on is a schema write, and a branch's Table classes carry the base's
4801
+ // logical name: without this a subscribe through a branched application would enable
4802
+ // auditing on the live base table for every other consumer, with no DDL call involved.
4803
+ TableResource.assertSchemaMutable('enable auditing for a subscription');
4059
4804
  table({ table: tableName, database: databaseName, schemaDefined, attributes, audit: true });
4060
4805
  }
4061
4806
  const getFullRecord = !request.rawEvents;
@@ -4110,7 +4855,7 @@ export function makeTable(options) {
4110
4855
  const subscription = addSubscription(
4111
4856
  TableResource,
4112
4857
  thisId,
4113
- function (id: Id, auditRecord?: any, localTime?: any, beginTxn?: any) {
4858
+ function (id: Id, auditRecord?: any, txnLogKey?: any, beginTxn?: any) {
4114
4859
  if (dropDuringReplay) return;
4115
4860
  try {
4116
4861
  let type = auditRecord.type;
@@ -4133,8 +4878,7 @@ export function makeTable(options) {
4133
4878
  // been written, so are fresh in memory.
4134
4879
  const entry: Entry = primaryStore.getEntry(id);
4135
4880
  if (entry) {
4136
- // staleness is a record-version comparison; auditRecord.version is the log key on RocksDB
4137
- if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version)) return; // out of order event, with old update, don't send anything
4881
+ if (entry.version !== auditRecord.version) return; // out of order event, with old update, don't send anything
4138
4882
  value = entry.value;
4139
4883
  type = entry.metadataFlags & INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
4140
4884
  } else {
@@ -4143,7 +4887,7 @@ export function makeTable(options) {
4143
4887
  }
4144
4888
  const event = {
4145
4889
  id,
4146
- localTime,
4890
+ localTime: txnLogKey,
4147
4891
  value,
4148
4892
  version: auditRecord.version,
4149
4893
  type,
@@ -4208,11 +4952,11 @@ export function makeTable(options) {
4208
4952
  if (auditRecord.tableId !== tableId) continue;
4209
4953
  const id = auditRecord.recordId;
4210
4954
  if (thisId == null || isDescendantId(thisId, id)) {
4211
- const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.localTime);
4955
+ const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
4212
4956
  if (
4213
4957
  !send({
4214
4958
  id,
4215
- localTime: auditRecord.localTime,
4959
+ localTime: auditRecord.txnLogKey,
4216
4960
  value,
4217
4961
  version: auditRecord.version,
4218
4962
  type: auditRecord.type,
@@ -4225,7 +4969,7 @@ export function makeTable(options) {
4225
4969
  if ((await subscription.waitForDrain()) === false) return;
4226
4970
  }
4227
4971
  }
4228
- subscription!.startTime = auditRecord.localTime ?? auditRecord.version; // update so we don't double send
4972
+ subscription!.startTime = auditRecord.txnLogKey; // update so we don't double send
4229
4973
  }
4230
4974
  } finally {
4231
4975
  // replay is done, we can start sending real-time messages again
@@ -4257,10 +5001,10 @@ export function makeTable(options) {
4257
5001
  );
4258
5002
  break;
4259
5003
  }
4260
- const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.localTime);
5004
+ const value = auditRecord.getValue(primaryStore, getFullRecord, auditRecord.txnLogKey);
4261
5005
  const historyEntry = {
4262
5006
  id,
4263
- localTime: auditRecord.localTime,
5007
+ localTime: auditRecord.txnLogKey,
4264
5008
  value,
4265
5009
  version: auditRecord.version,
4266
5010
  type: auditRecord.type,
@@ -4274,7 +5018,7 @@ export function makeTable(options) {
4274
5018
  if (--count <= 0) break;
4275
5019
  }
4276
5020
  } catch (error) {
4277
- logger.error?.('Error getting history entry', auditRecord.localTime, error);
5021
+ logger.error?.('Error getting history entry', auditRecord.txnLogKey, error);
4278
5022
  }
4279
5023
  }
4280
5024
  for (let i = history.length; i > 0;) {
@@ -4354,6 +5098,12 @@ export function makeTable(options) {
4354
5098
  logger.trace?.('re-retrieved record', localTime, this.#entry?.localTime);
4355
5099
  localTime = entry?.localTime;
4356
5100
  }
5101
+ let nodeId = entry?.nodeId;
5102
+ if (isRocksDB && entry) {
5103
+ const head = resolveAuditHead(thisId, entry.version, nodeId, entry.additionalAuditRefs);
5104
+ localTime = head.txnLogKey;
5105
+ nodeId = head.nodeId;
5106
+ }
4357
5107
  logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
4358
5108
  if (startTime < localTime) {
4359
5109
  // start time specified, get the audit history for this record. Set startTime up
@@ -4364,7 +5114,6 @@ export function makeTable(options) {
4364
5114
  const history = [];
4365
5115
  let inspected = 0;
4366
5116
  let nextTime = localTime;
4367
- let nodeId = entry?.nodeId;
4368
5117
  do {
4369
5118
  if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
4370
5119
  recordsSinceYield = 0;
@@ -4389,8 +5138,16 @@ export function makeTable(options) {
4389
5138
  if (count) count--;
4390
5139
  } else if (!isActive()) return;
4391
5140
  }
4392
- nextTime = auditRecord.previousVersion;
4393
- nodeId = auditRecord.previousNodeId;
5141
+ const previousHead = isRocksDB
5142
+ ? resolveAuditHead(
5143
+ thisId,
5144
+ auditRecord.previousVersion,
5145
+ auditRecord.previousNodeId,
5146
+ auditRecord.previousAdditionalAuditRefs
5147
+ )
5148
+ : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
5149
+ nextTime = previousHead.txnLogKey;
5150
+ nodeId = previousHead.nodeId;
4394
5151
  } else break;
4395
5152
  } while (nextTime > startTime && count !== 0);
4396
5153
  for (let i = history.length; i > 0;) {
@@ -4569,6 +5326,7 @@ export function makeTable(options) {
4569
5326
  store: primaryStore,
4570
5327
  entry: this.#entry,
4571
5328
  nodeName: (context as any)?.nodeName,
5329
+ recordVersion: options?.version,
4572
5330
  validate: () => {
4573
5331
  if (!(context as any)?.source) {
4574
5332
  transaction.checkOverloaded();
@@ -4830,6 +5588,7 @@ export function makeTable(options) {
4830
5588
  return this.#version;
4831
5589
  }
4832
5590
  static async addAttributes(attributesToAdd: Attribute[]) {
5591
+ TableResource.assertSchemaMutable('add attributes');
4833
5592
  const new_attributes = attributes.slice(0);
4834
5593
  for (const attribute of attributesToAdd) {
4835
5594
  if (!attribute.name) throw new ClientError('Attribute name is required');
@@ -4847,6 +5606,7 @@ export function makeTable(options) {
4847
5606
  return (TableResource as any).indexingOperation;
4848
5607
  }
4849
5608
  static async removeAttributes(names: string[]) {
5609
+ TableResource.assertSchemaMutable('remove attributes');
4850
5610
  const new_attributes = attributes.filter((attribute) => !names.includes(attribute.name));
4851
5611
  table({
4852
5612
  table: tableName,
@@ -4867,6 +5627,10 @@ export function makeTable(options) {
4867
5627
  const stats = primaryStore.getStats();
4868
5628
  return (stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize;
4869
5629
  }
5630
+ /** Sizes of this table's durable record-structure dictionaries. */
5631
+ static getStructureCounts(): StructureCounts | undefined {
5632
+ return primaryStore.encoder?.getStructureCounts?.();
5633
+ }
4870
5634
  static getAuditSize(): number {
4871
5635
  const stats = auditStore?.getStats();
4872
5636
  return (
@@ -5253,29 +6017,94 @@ export function makeTable(options) {
5253
6017
  this.userSetEmbedders.add(attribute_name);
5254
6018
  }
5255
6019
  static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise<number> {
5256
- let completion: Promise<void>;
6020
+ const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
6021
+ const inFlightRemovals = new Set<Promise<void>>();
6022
+ const removalSlotWaiters: Array<() => void> = [];
6023
+ let removalsAttempted = 0;
6024
+ let removalsSucceeded = 0;
6025
+ let firstRemovalError: unknown;
6026
+ function startRemoval(remove: () => MaybePromise<void>, errorMessage: string, onSuccess?: () => void): void {
6027
+ removalsAttempted++;
6028
+ const removal = new Promise<void>((resolve) => resolve(remove()))
6029
+ .then(
6030
+ () => {
6031
+ removalsSucceeded++;
6032
+ onSuccess?.();
6033
+ },
6034
+ (error) => {
6035
+ // capture before logging: a throwing logger must not cost us the error we may rethrow
6036
+ if (firstRemovalError === undefined) firstRemovalError = error;
6037
+ harperLogger.warn(errorMessage, error);
6038
+ }
6039
+ )
6040
+ .catch(() => undefined)
6041
+ .finally(() => {
6042
+ inFlightRemovals.delete(removal);
6043
+ removalSlotWaiters.shift()?.();
6044
+ });
6045
+ inFlightRemovals.add(removal);
6046
+ }
6047
+ function queueRemoval(
6048
+ remove: () => MaybePromise<void>,
6049
+ errorMessage: string,
6050
+ onSuccess?: () => void
6051
+ ): Promise<void> | undefined {
6052
+ if (inFlightRemovals.size >= maxConcurrentRemovals) {
6053
+ return new Promise<void>((resolve) => {
6054
+ removalSlotWaiters.push(resolve);
6055
+ }).then(() => startRemoval(remove, errorMessage, onSuccess));
6056
+ }
6057
+ startRemoval(remove, errorMessage, onSuccess);
6058
+ }
6059
+ const drainRemovals = () => Promise.all(inFlightRemovals);
5257
6060
  let entriesDeleted = 0;
5258
- for (const auditRecord of auditStore.getRange({
5259
- start: 0,
5260
- end: endTime,
5261
- })) {
5262
- await rest(); // yield to other async operations
5263
- if (auditRecord.tableId !== tableId) continue;
5264
- completion = removeAuditEntry(auditStore, auditRecord);
5265
- entriesDeleted++;
6061
+ try {
6062
+ for (const auditRecord of auditStore.getRange({
6063
+ start: 1, // must not be zero; see getHistory below for why
6064
+ end: endTime,
6065
+ })) {
6066
+ await rest(); // yield to other async operations
6067
+ if (auditRecord.tableId !== tableId) continue;
6068
+ const backpressure = queueRemoval(
6069
+ () => removeAuditEntry(auditStore, auditRecord),
6070
+ 'Error removing audit entry during deleteHistory',
6071
+ () => {
6072
+ entriesDeleted++;
6073
+ }
6074
+ );
6075
+ if (backpressure) await backpressure;
6076
+ }
6077
+ } finally {
6078
+ await drainRemovals();
5266
6079
  }
5267
6080
  if (cleanupDeletedRecords) {
5268
6081
  // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't
5269
6082
  // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records
5270
- for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
5271
- const { value, localTime } = entry;
5272
- await rest(); // yield to other async operations
5273
- if (value === null && localTime < endTime) {
5274
- completion = removeEntry(primaryStore, entry);
6083
+ try {
6084
+ for (const entry of primaryStore.getRange({ start: 0, versions: true })) {
6085
+ const { key, value, localTime, version } = entry;
6086
+ await rest(); // yield to other async operations
6087
+ const auditTime =
6088
+ isRocksDB && version != null
6089
+ ? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
6090
+ : localTime;
6091
+ if (value === null && version != null && auditTime < endTime) {
6092
+ const backpressure = queueRemoval(
6093
+ () => primaryStore.remove(key, version),
6094
+ 'Error removing deleted record during deleteHistory'
6095
+ );
6096
+ if (backpressure) await backpressure;
6097
+ }
5275
6098
  }
6099
+ } finally {
6100
+ await drainRemovals();
5276
6101
  }
5277
6102
  }
5278
- await completion;
6103
+ if (removalsAttempted > 0 && removalsSucceeded === 0) {
6104
+ // zero progress must not report the same success as "nothing was eligible" (see DESIGN.md);
6105
+ // partial failures stay best-effort, logged and excluded from the returned count
6106
+ throw firstRemovalError ?? new Error('Every removal attempted during deleteHistory failed');
6107
+ }
5279
6108
  return entriesDeleted;
5280
6109
  }
5281
6110
  static async *getHistory(startTime = 0, endTime = Infinity) {
@@ -5287,10 +6116,11 @@ export function makeTable(options) {
5287
6116
  if (auditRecord.tableId !== tableId) continue;
5288
6117
  yield {
5289
6118
  id: auditRecord.recordId,
5290
- localTime: auditRecord.version,
6119
+ // Compatibility-facing LMDB history has always reported/grouped by record version.
6120
+ localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
5291
6121
  version: auditRecord.version,
5292
6122
  type: auditRecord.type,
5293
- value: auditRecord.getValue(primaryStore, true, auditRecord.version),
6123
+ value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
5294
6124
  user: auditRecord.user,
5295
6125
  operation: auditRecord.originatingOperation,
5296
6126
  };
@@ -5301,7 +6131,9 @@ export function makeTable(options) {
5301
6131
  if (id == undefined) throw new Error('An id is required');
5302
6132
  const entry = primaryStore.getEntry(id);
5303
6133
  if (!entry) return history;
5304
- let nextVersion = entry.localTime;
6134
+ let nextVersion = isRocksDB
6135
+ ? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
6136
+ : entry.localTime;
5305
6137
  if (!nextVersion) throw new Error('The entry does not have a local audit time');
5306
6138
  const count = 0;
5307
6139
  const auditWindow = 100;
@@ -5314,17 +6146,25 @@ export function makeTable(options) {
5314
6146
  if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) {
5315
6147
  history.splice(insertionPoint, 0, {
5316
6148
  id: auditRecord.recordId,
5317
- localTime: auditRecord.version,
6149
+ localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
5318
6150
  version: auditRecord.version,
5319
6151
  type: auditRecord.type,
5320
- // reconstruct each entry's record image as of its own version, not the audit
6152
+ // reconstruct each entry's record image as of its own log position, not the audit
5321
6153
  // window boundary (nextVersion), matching getHistory (issue #1330)
5322
- value: auditRecord.getValue(primaryStore, true, auditRecord.version),
6154
+ value: auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey),
5323
6155
  user: auditRecord.user,
5324
6156
  operation: auditRecord.originatingOperation,
5325
6157
  });
5326
- if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) {
5327
- highestPreviousVersion = auditRecord.previousVersion;
6158
+ const previousVersion = isRocksDB
6159
+ ? resolveAuditHead(
6160
+ id,
6161
+ auditRecord.previousVersion,
6162
+ auditRecord.previousNodeId,
6163
+ auditRecord.previousAdditionalAuditRefs
6164
+ ).txnLogKey
6165
+ : auditRecord.previousVersion;
6166
+ if (previousVersion > highestPreviousVersion && previousVersion < start) {
6167
+ highestPreviousVersion = previousVersion;
5328
6168
  }
5329
6169
  }
5330
6170
  }
@@ -5343,8 +6183,14 @@ export function makeTable(options) {
5343
6183
  }
5344
6184
  return Promise.all(promises);
5345
6185
  }
6186
+ /** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
5346
6187
  static cleanup() {
6188
+ disposed = true;
6189
+ clearTimeout(cleanupTimer);
6190
+ settlePendingCleanup();
6191
+ clearInterval(recordExpirationInterval);
5347
6192
  deleteCallbackHandle?.remove();
6193
+ removeStorageReclamationHandler(primaryStore.path, reclamationHandler);
5348
6194
  }
5349
6195
  static _readTxnForContext(context) {
5350
6196
  return txnForContext(context).getReadTxn();
@@ -5366,9 +6212,14 @@ export function makeTable(options) {
5366
6212
  }
5367
6213
  );
5368
6214
 
5369
- TableResource.updatedAttributes(); // on creation, update accessors as well
5370
- if (expirationMs) TableResource.setTTLExpiration(expirationMs / 1000);
5371
- if (expiresAtProperty) runRecordExpirationEviction();
6215
+ try {
6216
+ TableResource.updatedAttributes(); // on creation, update accessors as well
6217
+ if (expirationMs) TableResource.setTTLExpiration(expirationMs / 1000);
6218
+ if (expiresAtProperty) runRecordExpirationEviction();
6219
+ } catch (error) {
6220
+ TableResource.cleanup();
6221
+ throw error;
6222
+ }
5372
6223
  return TableResource;
5373
6224
  function updateIndices(id: any, existingRecord: any, record: any, options?: any) {
5374
6225
  let hasChanges;
@@ -5775,6 +6626,22 @@ export function makeTable(options) {
5775
6626
  return transaction;
5776
6627
  }
5777
6628
  }
6629
+ /**
6630
+ * Detach an unsaved TransactionWrite that a scoped lock() eagerly staged (see #reloadLocked)
6631
+ * once its handle upgrades to hold: hold staging is deferred and explicit-save-only, so a
6632
+ * dangling scoped write would otherwise auto-commit at the transaction sweep and clobber
6633
+ * whatever the hold write lands. Marking it .dropped lets a later save() on the instance that
6634
+ * owns it (checked via #savingOperation === this write) fall through to the hold branch
6635
+ * instead of resolving a detached, dead reference.
6636
+ */
6637
+ function detachScopedUpgradeWrite(link: any, keyId: unknown, handle: RecordLockHandle): void {
6638
+ for (const write of link.writes) {
6639
+ if (write && !write.saved && write.lockHandle === handle && writeKeyId(write.key) === keyId) {
6640
+ write.dropped = true;
6641
+ link.detachWrite(write);
6642
+ }
6643
+ }
6644
+ }
5778
6645
  function getAttributeValue(entry, attribute_name, context, sort?) {
5779
6646
  if (!entry) {
5780
6647
  return;
@@ -5892,6 +6759,11 @@ export function makeTable(options) {
5892
6759
  const metadataFlags = existingEntry?.metadataFlags;
5893
6760
 
5894
6761
  const existingVersion = existingEntry?.version;
6762
+ const existingRecord = existingEntry?.value;
6763
+ const inheritedTimestamp = context?.timestamp || context?.transaction?.timestamp;
6764
+ const sourceTimestamp =
6765
+ inheritedTimestamp ||
6766
+ (isRocksDB ? (primaryStore as RocksDatabase).getMonotonicTimestamp() : getNextMonotonicTime());
5895
6767
  let whenResolved, timer;
5896
6768
  // We start by locking the record so that there is only one resolution happening at once;
5897
6769
  // if there is already a resolution in process, we want to use the results of that resolution
@@ -5930,10 +6802,8 @@ export function makeTable(options) {
5930
6802
  // lock acquired — this request will actually load from source
5931
6803
  setLoadedFromSource(target, true);
5932
6804
 
5933
- const existingRecord = existingEntry?.value;
5934
6805
  // it is important to remember that this is _NOT_ part of the current transaction; nothing is changing
5935
- // with the canonical data, we are simply fulfilling our local copy of the canonical data, but still don't
5936
- // want a timestamp later than the current transaction
6806
+ // with the canonical data, we are simply fulfilling our local copy of the canonical data.
5937
6807
  // we create a new context for the source, we want to determine the timestamp and don't want to
5938
6808
  // attribute this to the current user
5939
6809
  const sourceContext = {
@@ -5969,13 +6839,37 @@ export function makeTable(options) {
5969
6839
  // before the drain's fail-closed timeout below.
5970
6840
  const commitPromise = transaction(sourceContext, async (_txn) => {
5971
6841
  const start = performance.now();
5972
- let updatedRecord;
6842
+ let updatedRecord, assignCreatedTime, sourceVersion;
5973
6843
  let hasChanges, invalidated;
5974
6844
  try {
5975
6845
  updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
5976
6846
  invalidated = metadataFlags & INVALIDATED;
5977
- let version = sourceContext.lastModified || (invalidated && existingVersion);
5978
- hasChanges = invalidated || version > existingVersion || !existingRecord;
6847
+ const reportedVersion = sourceContext.lastModified;
6848
+ const validReportedVersion =
6849
+ typeof reportedVersion === 'number' &&
6850
+ Number.isFinite(reportedVersion) &&
6851
+ reportedVersion > 0 &&
6852
+ reportedVersion <= MAX_DATE_TIMESTAMP;
6853
+ if (validReportedVersion) {
6854
+ // A record version is also this node's ordering token (precedesExistingVersion), so a
6855
+ // source-reported version ahead of local time would make every subsequent local write look
6856
+ // out-of-order and be discarded until wall-clock caught up — freezing the row. Honor what
6857
+ // the source reports, but never beyond now.
6858
+ const versionCeiling = Math.max(sourceTimestamp, Date.now());
6859
+ sourceVersion = Math.min(reportedVersion, versionCeiling);
6860
+ if (sourceVersion !== reportedVersion) {
6861
+ logger.trace?.(
6862
+ `Capping future source version for ${tableName} id ${id}: ${reportedVersion} -> ${sourceVersion}`
6863
+ );
6864
+ if (!warnedFutureSourceVersion) {
6865
+ warnedFutureSourceVersion = true;
6866
+ logger.warn?.(
6867
+ `The source for ${tableName} reported a lastModified ahead of local time (${new Date(reportedVersion).toISOString()}) for id ${id}; capping cached record versions at local time`
6868
+ );
6869
+ }
6870
+ }
6871
+ } else sourceVersion = sourceTimestamp;
6872
+ hasChanges = invalidated || (validReportedVersion && reportedVersion > existingVersion) || !existingRecord;
5979
6873
  const resolveDuration = performance.now() - start;
5980
6874
  recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success');
5981
6875
  if (responseHeaders)
@@ -5989,7 +6883,7 @@ export function makeTable(options) {
5989
6883
  if (status === 304) {
5990
6884
  // revalidation of our current cached record
5991
6885
  updatedRecord = existingRecord;
5992
- version = existingVersion;
6886
+ sourceVersion = existingVersion;
5993
6887
  } else if (!CACHEABLE_STATUS_CODES.has(status)) {
5994
6888
  // non-cacheable status - propagate to client without caching
5995
6889
  throw new ServerError(updatedRecord.body || 'Error from source', status);
@@ -6059,10 +6953,15 @@ export function makeTable(options) {
6059
6953
  updatedRecord = storedFieldsOnly(primaryStore.encoder, updatedRecord);
6060
6954
  if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
6061
6955
  }
6956
+ assignCreatedTime = createdTimeProperty && updatedRecord?.[createdTimeProperty.name] == null;
6062
6957
  resolved = true;
6958
+ const resolvedVersion =
6959
+ isRocksDB && updatedRecord && existingVersion != null
6960
+ ? Math.max(sourceVersion, existingVersion)
6961
+ : sourceVersion;
6063
6962
  const resolvedEntry: Entry = {
6064
6963
  key: id,
6065
- version,
6964
+ version: resolvedVersion,
6066
6965
  value: updatedRecord,
6067
6966
  expiresAt: sourceContext.expiresAt,
6068
6967
  metadataFlags: 0,
@@ -6119,16 +7018,33 @@ export function makeTable(options) {
6119
7018
  const sourceWrite: any = {
6120
7019
  key: id,
6121
7020
  store: primaryStore,
6122
- entry: existingEntry,
7021
+ entry: undefined,
6123
7022
  nodeName: 'source',
6124
- commit: (txnTime, existingEntry, _retry, transaction: any) => {
7023
+ commit: (_txnTime, existingEntry, _retry, transaction: any) => {
6125
7024
  sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
6126
- if (existingEntry?.version !== existingVersion) {
6127
- // don't do anything if the version has changed
7025
+ const racedVersion = existingEntry?.version;
7026
+ // A first fill may replace a record that raced it only when its candidate version strictly
7027
+ // orders after that record. The comparison has to be replica-independent, so a tie leaves the
7028
+ // raced record in place: precedesExistingVersion() would break the tie with *this* node's
7029
+ // name, and a fill from a shared source has no node identity of its own, so two replicas
7030
+ // resolving the same tie could keep different values at the same version.
7031
+ const replacesRacedRecord = racedVersion == null || sourceVersion > racedVersion;
7032
+ if (
7033
+ racedVersion !== existingVersion &&
7034
+ // Revalidations retain exact-CAS semantics; first fills use deterministic ordering.
7035
+ (existingVersion != null || !updatedRecord || !replacesRacedRecord)
7036
+ ) {
7037
+ logger.trace?.(
7038
+ `Discarding resolved record from source with id: ${id}, source version: ${sourceVersion}, current version: ${racedVersion}`
7039
+ );
6128
7040
  sourceWrite.skipped = true;
6129
7041
  return;
6130
7042
  }
6131
- updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
7043
+ const currentRecord = existingEntry?.value;
7044
+ const recordVersion =
7045
+ isRocksDB && racedVersion != null ? Math.max(sourceVersion, racedVersion) : sourceVersion;
7046
+ const txnLogKey = isRocksDB ? transaction?.getTimestamp?.() : recordVersion;
7047
+ updateIndices(id, currentRecord, updatedRecord, transaction && { transaction });
6132
7048
  if (updatedRecord) {
6133
7049
  if (existingEntry) {
6134
7050
  context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
@@ -6139,22 +7055,22 @@ export function makeTable(options) {
6139
7055
  if (updatedTimeProperty) {
6140
7056
  updatedRecord[updatedTimeProperty.name] =
6141
7057
  updatedTimeProperty.type === 'Date'
6142
- ? new Date(txnTime)
7058
+ ? new Date(recordVersion)
6143
7059
  : updatedTimeProperty.type === 'String'
6144
- ? new Date(txnTime).toISOString()
6145
- : txnTime;
7060
+ ? new Date(recordVersion).toISOString()
7061
+ : recordVersion;
6146
7062
  }
6147
- if (createdTimeProperty && updatedRecord[createdTimeProperty.name] == null) {
6148
- const existingCreatedTime = existingEntry?.value?.[createdTimeProperty.name];
7063
+ if (assignCreatedTime) {
7064
+ const existingCreatedTime = currentRecord?.[createdTimeProperty.name];
6149
7065
  if (existingCreatedTime != null) {
6150
7066
  updatedRecord[createdTimeProperty.name] = existingCreatedTime;
6151
7067
  } else {
6152
7068
  updatedRecord[createdTimeProperty.name] =
6153
7069
  createdTimeProperty.type === 'Date'
6154
- ? new Date(txnTime)
7070
+ ? new Date(recordVersion)
6155
7071
  : createdTimeProperty.type === 'String'
6156
- ? new Date(txnTime).toISOString()
6157
- : txnTime;
7072
+ ? new Date(recordVersion).toISOString()
7073
+ : recordVersion;
6158
7074
  }
6159
7075
  }
6160
7076
  const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
@@ -6186,22 +7102,25 @@ export function makeTable(options) {
6186
7102
  residencyId = getResidencyId(residency);
6187
7103
  }
6188
7104
  logger.trace?.(
6189
- `Writing resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
7105
+ `Writing resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`
6190
7106
  );
6191
7107
  // TODO: We are doing a double check for ifVersion that should probably be cleaned out
7108
+ const writeAudit = (audit && (hasChanges || omitLocalRecord)) || null;
6192
7109
  updateRecord(
6193
7110
  id,
6194
7111
  updatedRecord,
6195
7112
  existingEntry,
6196
- txnTime,
7113
+ recordVersion,
6197
7114
  omitLocalRecord ? INVALIDATED : 0,
6198
- (audit && (hasChanges || omitLocalRecord)) || null,
7115
+ writeAudit,
6199
7116
  {
6200
7117
  user: (sourceContext as any)?.user,
6201
7118
  expiresAt: sourceContext.expiresAt,
6202
7119
  residencyId,
6203
7120
  transaction,
6204
7121
  tableToTrack: tableName,
7122
+ additionalAuditRefs:
7123
+ writeAudit && txnLogKey !== recordVersion ? [{ version: txnLogKey, nodeId: 0 }] : undefined,
6205
7124
  },
6206
7125
  'put',
6207
7126
  Boolean(invalidated),
@@ -6211,17 +7130,26 @@ export function makeTable(options) {
6211
7130
  if (sourceContext.expiresAt) scheduleCleanup();
6212
7131
  } else if (existingEntry) {
6213
7132
  logger.trace?.(
6214
- `Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
7133
+ `Deleting resolved record from source with id: ${id}, timestamp: ${new Date(recordVersion).toISOString()}`
6215
7134
  );
6216
7135
  if (audit || trackDeletes) {
6217
7136
  updateRecord(
6218
7137
  id,
6219
7138
  null,
6220
7139
  existingEntry,
6221
- txnTime,
7140
+ recordVersion,
6222
7141
  0,
6223
7142
  (audit && hasChanges) || null,
6224
- { user: (sourceContext as any)?.user, transaction, tableToTrack: tableName },
7143
+ {
7144
+ user: (sourceContext as any)?.user,
7145
+ transaction,
7146
+ tableToTrack: tableName,
7147
+ recordVersion,
7148
+ additionalAuditRefs:
7149
+ audit && hasChanges && txnLogKey !== recordVersion
7150
+ ? [{ version: txnLogKey, nodeId: 0 }]
7151
+ : undefined,
7152
+ },
6225
7153
  'delete',
6226
7154
  Boolean(invalidated)
6227
7155
  );
@@ -6384,7 +7312,13 @@ export function makeTable(options) {
6384
7312
  },
6385
7313
  };
6386
7314
  }
7315
+ function settlePendingCleanup() {
7316
+ for (const resolve of pendingCleanupResolvers) resolve();
7317
+ pendingCleanupResolvers.clear();
7318
+ }
6387
7319
  function scheduleCleanup(priority?: number): Promise<void> | void {
7320
+ // a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
7321
+ if (disposed) return;
6388
7322
  let runImmediately = false;
6389
7323
  if (priority) {
6390
7324
  // run immediately if there is a big increase in priority
@@ -6397,8 +7331,18 @@ export function makeTable(options) {
6397
7331
  if (getWorkerIndex() === getWorkerCount() - 1) {
6398
7332
  // run on the last thread so we aren't overloading lower-numbered threads
6399
7333
  if (cleanupTimer) clearTimeout(cleanupTimer);
6400
- if (!cleanupInterval) return;
6401
- return new Promise((resolve) => {
7334
+ if (!cleanupInterval) {
7335
+ // no replacement pass is being scheduled, so nothing is left to settle a superseded one
7336
+ settlePendingCleanup();
7337
+ return;
7338
+ }
7339
+ // This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
7340
+ // this pass's scan completes, so a reclamation run is never told the storage was reclaimed
7341
+ // before any scan ran. It has to run now, though — that run blocks its whole path on the
7342
+ // promise, and the replacement's own slot can be a full interval out.
7343
+ if (pendingCleanupResolvers.size > 0) runImmediately = true;
7344
+ return new Promise<void>((resolve) => {
7345
+ pendingCleanupResolvers.add(resolve);
6402
7346
  const startOfYear = new Date();
6403
7347
  startOfYear.setMonth(0);
6404
7348
  startOfYear.setDate(1);
@@ -6411,6 +7355,7 @@ export function makeTable(options) {
6411
7355
  ? Date.now()
6412
7356
  : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
6413
7357
  const startNextTimer = (nextScheduled) => {
7358
+ if (disposed) return;
6414
7359
  logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
6415
7360
  // noinspection JSVoidFunctionReturnValueUsed
6416
7361
  cleanupTimer = setTimeout(
@@ -6421,8 +7366,11 @@ export function makeTable(options) {
6421
7366
  const rootStore = primaryStore.rootStore;
6422
7367
  if (rootStore.status !== 'open') {
6423
7368
  clearTimeout(cleanupTimer);
7369
+ settlePendingCleanup();
6424
7370
  return;
6425
7371
  }
7372
+ // snapshot: an awaiter that arrives during this scan belongs to the pass that supersedes it
7373
+ const settling = [...pendingCleanupResolvers];
6426
7374
  const MAX_CLEANUP_CONCURRENCY = 50;
6427
7375
  const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY);
6428
7376
  let cleanupIndex = 0;
@@ -6502,7 +7450,10 @@ export function makeTable(options) {
6502
7450
  } catch (error) {
6503
7451
  logger.warn?.(`Error in cleanup scan for ${tableName}:`, error);
6504
7452
  }
6505
- resolve(undefined);
7453
+ for (const settle of settling) {
7454
+ pendingCleanupResolvers.delete(settle);
7455
+ settle();
7456
+ }
6506
7457
  cleanupPriority = 0; // reset the priority
6507
7458
  })),
6508
7459
  Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number
@@ -6514,17 +7465,17 @@ export function makeTable(options) {
6514
7465
  }
6515
7466
  function addDeleteRemoval() {
6516
7467
  deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id: Id, version: number) => {
6517
- primaryStore.remove(id, version);
7468
+ return primaryStore.remove(id, version);
6518
7469
  });
6519
7470
  }
6520
7471
  function runRecordExpirationEviction() {
6521
7472
  // Periodically evict expired records, searching for records who expiresAt timestamp is before now
6522
7473
  if (getWorkerIndex() === 0) {
6523
7474
  // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
6524
- setInterval(async () => {
7475
+ recordExpirationInterval = setInterval(async () => {
6525
7476
  // go through each database and table and then search for expired entries
6526
7477
  // find any entries that are set to expire before now
6527
- if (runningRecordExpiration) return;
7478
+ if (disposed || runningRecordExpiration) return;
6528
7479
  runningRecordExpiration = true;
6529
7480
  try {
6530
7481
  const expiresAtName = expiresAtProperty.name;