@powersync/service-core 0.8.8 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (372) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/dist/api/RouteAPI.d.ts +67 -0
  3. package/dist/api/RouteAPI.js +2 -0
  4. package/dist/api/RouteAPI.js.map +1 -0
  5. package/dist/api/api-index.d.ts +1 -0
  6. package/dist/api/api-index.js +1 -0
  7. package/dist/api/api-index.js.map +1 -1
  8. package/dist/api/diagnostics.d.ts +4 -4
  9. package/dist/api/diagnostics.js +170 -158
  10. package/dist/api/diagnostics.js.map +1 -1
  11. package/dist/api/schema.d.ts +3 -5
  12. package/dist/api/schema.js +14 -80
  13. package/dist/api/schema.js.map +1 -1
  14. package/dist/auth/CachedKeyCollector.js.map +1 -1
  15. package/dist/auth/KeySpec.js.map +1 -1
  16. package/dist/auth/KeyStore.d.ts +7 -4
  17. package/dist/auth/KeyStore.js +1 -1
  18. package/dist/auth/KeyStore.js.map +1 -1
  19. package/dist/auth/LeakyBucket.js.map +1 -1
  20. package/dist/auth/RemoteJWKSCollector.d.ts +0 -2
  21. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  22. package/dist/auth/auth-index.d.ts +0 -1
  23. package/dist/auth/auth-index.js +0 -1
  24. package/dist/auth/auth-index.js.map +1 -1
  25. package/dist/db/mongo.js +5 -3
  26. package/dist/db/mongo.js.map +1 -1
  27. package/dist/entry/cli-entry.js +3 -2
  28. package/dist/entry/cli-entry.js.map +1 -1
  29. package/dist/entry/commands/compact-action.js +90 -14
  30. package/dist/entry/commands/compact-action.js.map +1 -1
  31. package/dist/entry/commands/migrate-action.js +4 -5
  32. package/dist/entry/commands/migrate-action.js.map +1 -1
  33. package/dist/entry/commands/teardown-action.js +2 -2
  34. package/dist/entry/commands/teardown-action.js.map +1 -1
  35. package/dist/index.d.ts +4 -2
  36. package/dist/index.js +4 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/locks/MongoLocks.js.map +1 -1
  39. package/dist/metrics/Metrics.d.ts +2 -2
  40. package/dist/metrics/Metrics.js +5 -13
  41. package/dist/metrics/Metrics.js.map +1 -1
  42. package/dist/migrations/db/migrations/1684951997326-init.d.ts +2 -2
  43. package/dist/migrations/db/migrations/1684951997326-init.js +4 -2
  44. package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -1
  45. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +2 -2
  46. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +4 -2
  47. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
  48. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +2 -2
  49. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +4 -2
  50. package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -1
  51. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts +3 -0
  52. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js +31 -0
  53. package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map +1 -0
  54. package/dist/migrations/executor.js.map +1 -1
  55. package/dist/migrations/migrations.d.ts +8 -0
  56. package/dist/migrations/migrations.js +19 -7
  57. package/dist/migrations/migrations.js.map +1 -1
  58. package/dist/migrations/store/migration-store.js.map +1 -1
  59. package/dist/modules/AbstractModule.d.ts +26 -0
  60. package/dist/modules/AbstractModule.js +11 -0
  61. package/dist/modules/AbstractModule.js.map +1 -0
  62. package/dist/modules/ModuleManager.d.ts +11 -0
  63. package/dist/modules/ModuleManager.js +32 -0
  64. package/dist/modules/ModuleManager.js.map +1 -0
  65. package/dist/modules/modules-index.d.ts +2 -0
  66. package/dist/modules/modules-index.js +3 -0
  67. package/dist/modules/modules-index.js.map +1 -0
  68. package/dist/replication/AbstractReplicationJob.d.ts +37 -0
  69. package/dist/replication/AbstractReplicationJob.js +51 -0
  70. package/dist/replication/AbstractReplicationJob.js.map +1 -0
  71. package/dist/replication/AbstractReplicator.d.ts +53 -0
  72. package/dist/replication/AbstractReplicator.js +250 -0
  73. package/dist/replication/AbstractReplicator.js.map +1 -0
  74. package/dist/replication/ErrorRateLimiter.d.ts +0 -10
  75. package/dist/replication/ErrorRateLimiter.js +1 -42
  76. package/dist/replication/ErrorRateLimiter.js.map +1 -1
  77. package/dist/replication/ReplicationEngine.d.ts +18 -0
  78. package/dist/replication/ReplicationEngine.js +41 -0
  79. package/dist/replication/ReplicationEngine.js.map +1 -0
  80. package/dist/replication/ReplicationModule.d.ts +51 -0
  81. package/dist/replication/ReplicationModule.js +68 -0
  82. package/dist/replication/ReplicationModule.js.map +1 -0
  83. package/dist/replication/replication-index.d.ts +4 -6
  84. package/dist/replication/replication-index.js +4 -6
  85. package/dist/replication/replication-index.js.map +1 -1
  86. package/dist/routes/RouterEngine.d.ts +42 -0
  87. package/dist/routes/RouterEngine.js +80 -0
  88. package/dist/routes/RouterEngine.js.map +1 -0
  89. package/dist/routes/auth.d.ts +2 -2
  90. package/dist/routes/auth.js +11 -11
  91. package/dist/routes/auth.js.map +1 -1
  92. package/dist/routes/configure-fastify.d.ts +37 -23
  93. package/dist/routes/configure-fastify.js +18 -18
  94. package/dist/routes/configure-fastify.js.map +1 -1
  95. package/dist/routes/configure-rsocket.d.ts +3 -4
  96. package/dist/routes/configure-rsocket.js +7 -4
  97. package/dist/routes/configure-rsocket.js.map +1 -1
  98. package/dist/routes/endpoints/admin.d.ts +30 -0
  99. package/dist/routes/endpoints/admin.js +46 -67
  100. package/dist/routes/endpoints/admin.js.map +1 -1
  101. package/dist/routes/endpoints/checkpointing.js +103 -15
  102. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  103. package/dist/routes/endpoints/socket-route.js +8 -6
  104. package/dist/routes/endpoints/socket-route.js.map +1 -1
  105. package/dist/routes/endpoints/sync-rules.d.ts +1 -1
  106. package/dist/routes/endpoints/sync-rules.js +32 -23
  107. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  108. package/dist/routes/endpoints/sync-stream.d.ts +0 -1
  109. package/dist/routes/endpoints/sync-stream.js +8 -8
  110. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  111. package/dist/routes/hooks.js.map +1 -1
  112. package/dist/routes/route-register.js.map +1 -1
  113. package/dist/routes/router.d.ts +9 -2
  114. package/dist/routes/router.js.map +1 -1
  115. package/dist/routes/routes-index.d.ts +1 -0
  116. package/dist/routes/routes-index.js +1 -0
  117. package/dist/routes/routes-index.js.map +1 -1
  118. package/dist/runner/teardown.js +109 -76
  119. package/dist/runner/teardown.js.map +1 -1
  120. package/dist/storage/BucketStorage.d.ts +86 -36
  121. package/dist/storage/BucketStorage.js +6 -10
  122. package/dist/storage/BucketStorage.js.map +1 -1
  123. package/dist/storage/ChecksumCache.js.map +1 -1
  124. package/dist/storage/MongoBucketStorage.d.ts +7 -11
  125. package/dist/storage/MongoBucketStorage.js +48 -41
  126. package/dist/storage/MongoBucketStorage.js.map +1 -1
  127. package/dist/storage/ReplicationEventPayload.d.ts +14 -0
  128. package/dist/storage/ReplicationEventPayload.js +2 -0
  129. package/dist/storage/ReplicationEventPayload.js.map +1 -0
  130. package/dist/storage/SourceEntity.d.ts +20 -0
  131. package/dist/storage/SourceEntity.js +2 -0
  132. package/dist/storage/SourceEntity.js.map +1 -0
  133. package/dist/storage/SourceTable.d.ts +12 -5
  134. package/dist/storage/SourceTable.js +12 -5
  135. package/dist/storage/SourceTable.js.map +1 -1
  136. package/dist/storage/StorageEngine.d.ts +28 -0
  137. package/dist/storage/StorageEngine.js +45 -0
  138. package/dist/storage/StorageEngine.js.map +1 -0
  139. package/dist/storage/StorageProvider.d.ts +21 -0
  140. package/dist/storage/StorageProvider.js +2 -0
  141. package/dist/storage/StorageProvider.js.map +1 -0
  142. package/dist/storage/WriteCheckpointAPI.d.ts +74 -0
  143. package/dist/storage/WriteCheckpointAPI.js +16 -0
  144. package/dist/storage/WriteCheckpointAPI.js.map +1 -0
  145. package/dist/storage/mongo/MongoBucketBatch.d.ts +24 -5
  146. package/dist/storage/mongo/MongoBucketBatch.js +119 -62
  147. package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
  148. package/dist/storage/mongo/MongoCompactor.js +20 -3
  149. package/dist/storage/mongo/MongoCompactor.js.map +1 -1
  150. package/dist/storage/mongo/MongoIdSequence.js.map +1 -1
  151. package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +2 -2
  152. package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +2 -2
  153. package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -1
  154. package/dist/storage/mongo/MongoStorageProvider.d.ts +5 -0
  155. package/dist/storage/mongo/MongoStorageProvider.js +26 -0
  156. package/dist/storage/mongo/MongoStorageProvider.js.map +1 -0
  157. package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +18 -10
  158. package/dist/storage/mongo/MongoSyncBucketStorage.js +140 -25
  159. package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
  160. package/dist/storage/mongo/MongoSyncRulesLock.js +1 -1
  161. package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -1
  162. package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +20 -0
  163. package/dist/storage/mongo/MongoWriteCheckpointAPI.js +103 -0
  164. package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -0
  165. package/dist/storage/mongo/OperationBatch.d.ts +13 -4
  166. package/dist/storage/mongo/OperationBatch.js +25 -7
  167. package/dist/storage/mongo/OperationBatch.js.map +1 -1
  168. package/dist/storage/mongo/PersistedBatch.d.ts +3 -3
  169. package/dist/storage/mongo/PersistedBatch.js +2 -2
  170. package/dist/storage/mongo/PersistedBatch.js.map +1 -1
  171. package/dist/storage/mongo/config.d.ts +19 -0
  172. package/dist/storage/mongo/config.js +26 -0
  173. package/dist/storage/mongo/config.js.map +1 -0
  174. package/dist/storage/mongo/db.d.ts +3 -2
  175. package/dist/storage/mongo/db.js +1 -0
  176. package/dist/storage/mongo/db.js.map +1 -1
  177. package/dist/storage/mongo/models.d.ts +20 -5
  178. package/dist/storage/mongo/models.js.map +1 -1
  179. package/dist/storage/mongo/util.d.ts +12 -1
  180. package/dist/storage/mongo/util.js +50 -2
  181. package/dist/storage/mongo/util.js.map +1 -1
  182. package/dist/storage/storage-index.d.ts +8 -2
  183. package/dist/storage/storage-index.js +8 -2
  184. package/dist/storage/storage-index.js.map +1 -1
  185. package/dist/sync/BroadcastIterable.d.ts +0 -1
  186. package/dist/sync/BroadcastIterable.js.map +1 -1
  187. package/dist/sync/LastValueSink.d.ts +0 -1
  188. package/dist/sync/LastValueSink.js.map +1 -1
  189. package/dist/sync/merge.d.ts +0 -1
  190. package/dist/sync/merge.js.map +1 -1
  191. package/dist/sync/safeRace.js.map +1 -1
  192. package/dist/sync/sync.d.ts +1 -1
  193. package/dist/sync/sync.js +5 -5
  194. package/dist/sync/sync.js.map +1 -1
  195. package/dist/sync/util.d.ts +0 -2
  196. package/dist/sync/util.js.map +1 -1
  197. package/dist/system/ServiceContext.d.ts +37 -0
  198. package/dist/system/ServiceContext.js +48 -0
  199. package/dist/system/ServiceContext.js.map +1 -0
  200. package/dist/system/system-index.d.ts +1 -1
  201. package/dist/system/system-index.js +1 -1
  202. package/dist/system/system-index.js.map +1 -1
  203. package/dist/util/Mutex.js.map +1 -1
  204. package/dist/util/config/collectors/config-collector.js.map +1 -1
  205. package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
  206. package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
  207. package/dist/util/config/compound-config-collector.d.ts +9 -2
  208. package/dist/util/config/compound-config-collector.js +16 -24
  209. package/dist/util/config/compound-config-collector.js.map +1 -1
  210. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  211. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  212. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
  213. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +9 -0
  214. package/dist/util/config/sync-rules/sync-rules-provider.js +15 -0
  215. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -0
  216. package/dist/util/config/types.d.ts +7 -4
  217. package/dist/util/config/types.js.map +1 -1
  218. package/dist/util/config.d.ts +3 -4
  219. package/dist/util/config.js +5 -20
  220. package/dist/util/config.js.map +1 -1
  221. package/dist/util/memory-tracking.js.map +1 -1
  222. package/dist/util/secs.js.map +1 -1
  223. package/dist/util/util-index.d.ts +3 -6
  224. package/dist/util/util-index.js +3 -6
  225. package/dist/util/util-index.js.map +1 -1
  226. package/dist/util/utils.d.ts +10 -7
  227. package/dist/util/utils.js +36 -25
  228. package/dist/util/utils.js.map +1 -1
  229. package/package.json +8 -12
  230. package/src/api/RouteAPI.ts +78 -0
  231. package/src/api/api-index.ts +1 -0
  232. package/src/api/diagnostics.ts +18 -70
  233. package/src/api/schema.ts +18 -90
  234. package/src/auth/KeyStore.ts +9 -6
  235. package/src/auth/RemoteJWKSCollector.ts +4 -1
  236. package/src/auth/auth-index.ts +0 -1
  237. package/src/db/mongo.ts +5 -3
  238. package/src/entry/cli-entry.ts +3 -2
  239. package/src/entry/commands/compact-action.ts +24 -12
  240. package/src/entry/commands/migrate-action.ts +5 -8
  241. package/src/entry/commands/teardown-action.ts +2 -2
  242. package/src/index.ts +5 -2
  243. package/src/metrics/Metrics.ts +6 -16
  244. package/src/migrations/db/migrations/1684951997326-init.ts +9 -4
  245. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -4
  246. package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +6 -4
  247. package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +37 -0
  248. package/src/migrations/migrations.ts +24 -8
  249. package/src/modules/AbstractModule.ts +37 -0
  250. package/src/modules/ModuleManager.ts +34 -0
  251. package/src/modules/modules-index.ts +2 -0
  252. package/src/replication/AbstractReplicationJob.ts +79 -0
  253. package/src/replication/AbstractReplicator.ts +228 -0
  254. package/src/replication/ErrorRateLimiter.ts +0 -44
  255. package/src/replication/ReplicationEngine.ts +43 -0
  256. package/src/replication/ReplicationModule.ts +122 -0
  257. package/src/replication/replication-index.ts +4 -6
  258. package/src/routes/RouterEngine.ts +120 -0
  259. package/src/routes/auth.ts +21 -12
  260. package/src/routes/configure-fastify.ts +26 -27
  261. package/src/routes/configure-rsocket.ts +13 -8
  262. package/src/routes/endpoints/admin.ts +72 -76
  263. package/src/routes/endpoints/checkpointing.ts +51 -11
  264. package/src/routes/endpoints/socket-route.ts +10 -6
  265. package/src/routes/endpoints/sync-rules.ts +41 -25
  266. package/src/routes/endpoints/sync-stream.ts +8 -8
  267. package/src/routes/router.ts +8 -3
  268. package/src/routes/routes-index.ts +1 -0
  269. package/src/runner/teardown.ts +50 -88
  270. package/src/storage/BucketStorage.ts +103 -41
  271. package/src/storage/MongoBucketStorage.ts +65 -53
  272. package/src/storage/ReplicationEventPayload.ts +16 -0
  273. package/src/storage/SourceEntity.ts +22 -0
  274. package/src/storage/SourceTable.ts +14 -7
  275. package/src/storage/StorageEngine.ts +62 -0
  276. package/src/storage/StorageProvider.ts +27 -0
  277. package/src/storage/WriteCheckpointAPI.ts +85 -0
  278. package/src/storage/mongo/MongoBucketBatch.ts +164 -84
  279. package/src/storage/mongo/MongoCompactor.ts +25 -4
  280. package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +7 -4
  281. package/src/storage/mongo/MongoStorageProvider.ts +31 -0
  282. package/src/storage/mongo/MongoSyncBucketStorage.ts +118 -41
  283. package/src/storage/mongo/MongoSyncRulesLock.ts +7 -3
  284. package/src/storage/mongo/MongoWriteCheckpointAPI.ts +151 -0
  285. package/src/storage/mongo/OperationBatch.ts +28 -12
  286. package/src/storage/mongo/PersistedBatch.ts +10 -6
  287. package/src/storage/mongo/config.ts +40 -0
  288. package/src/storage/mongo/db.ts +4 -1
  289. package/src/storage/mongo/models.ts +21 -5
  290. package/src/storage/mongo/util.ts +48 -3
  291. package/src/storage/storage-index.ts +8 -2
  292. package/src/sync/sync.ts +7 -4
  293. package/src/sync/util.ts +0 -1
  294. package/src/system/ServiceContext.ts +68 -0
  295. package/src/system/system-index.ts +1 -1
  296. package/src/util/config/compound-config-collector.ts +31 -31
  297. package/src/util/config/sync-rules/sync-rules-provider.ts +18 -0
  298. package/src/util/config/types.ts +7 -5
  299. package/src/util/config.ts +6 -23
  300. package/src/util/util-index.ts +3 -6
  301. package/src/util/utils.ts +48 -41
  302. package/test/src/__snapshots__/sync.test.ts.snap +14 -14
  303. package/test/src/auth.test.ts +7 -7
  304. package/test/src/broadcast_iterable.test.ts +1 -1
  305. package/test/src/compacting.test.ts +50 -40
  306. package/test/src/data_storage.test.ts +382 -202
  307. package/test/src/env.ts +1 -3
  308. package/test/src/merge_iterable.test.ts +1 -6
  309. package/test/src/routes/probes.integration.test.ts +34 -30
  310. package/test/src/setup.ts +1 -1
  311. package/test/src/stream_utils.ts +42 -0
  312. package/test/src/sync.test.ts +115 -39
  313. package/test/src/util.ts +48 -51
  314. package/test/tsconfig.json +1 -1
  315. package/tsconfig.tsbuildinfo +1 -1
  316. package/vitest.config.ts +7 -1
  317. package/dist/auth/SupabaseKeyCollector.d.ts +0 -22
  318. package/dist/auth/SupabaseKeyCollector.js +0 -61
  319. package/dist/auth/SupabaseKeyCollector.js.map +0 -1
  320. package/dist/replication/PgRelation.d.ts +0 -16
  321. package/dist/replication/PgRelation.js +0 -26
  322. package/dist/replication/PgRelation.js.map +0 -1
  323. package/dist/replication/WalConnection.d.ts +0 -34
  324. package/dist/replication/WalConnection.js +0 -190
  325. package/dist/replication/WalConnection.js.map +0 -1
  326. package/dist/replication/WalStream.d.ts +0 -57
  327. package/dist/replication/WalStream.js +0 -519
  328. package/dist/replication/WalStream.js.map +0 -1
  329. package/dist/replication/WalStreamManager.d.ts +0 -30
  330. package/dist/replication/WalStreamManager.js +0 -198
  331. package/dist/replication/WalStreamManager.js.map +0 -1
  332. package/dist/replication/WalStreamRunner.d.ts +0 -38
  333. package/dist/replication/WalStreamRunner.js +0 -155
  334. package/dist/replication/WalStreamRunner.js.map +0 -1
  335. package/dist/replication/util.d.ts +0 -9
  336. package/dist/replication/util.js +0 -62
  337. package/dist/replication/util.js.map +0 -1
  338. package/dist/system/CorePowerSyncSystem.d.ts +0 -23
  339. package/dist/system/CorePowerSyncSystem.js +0 -52
  340. package/dist/system/CorePowerSyncSystem.js.map +0 -1
  341. package/dist/util/PgManager.d.ts +0 -24
  342. package/dist/util/PgManager.js +0 -55
  343. package/dist/util/PgManager.js.map +0 -1
  344. package/dist/util/migration_lib.d.ts +0 -11
  345. package/dist/util/migration_lib.js +0 -64
  346. package/dist/util/migration_lib.js.map +0 -1
  347. package/dist/util/pgwire_utils.d.ts +0 -24
  348. package/dist/util/pgwire_utils.js +0 -117
  349. package/dist/util/pgwire_utils.js.map +0 -1
  350. package/dist/util/populate_test_data.d.ts +0 -8
  351. package/dist/util/populate_test_data.js +0 -65
  352. package/dist/util/populate_test_data.js.map +0 -1
  353. package/src/auth/SupabaseKeyCollector.ts +0 -67
  354. package/src/replication/PgRelation.ts +0 -42
  355. package/src/replication/WalConnection.ts +0 -227
  356. package/src/replication/WalStream.ts +0 -631
  357. package/src/replication/WalStreamManager.ts +0 -213
  358. package/src/replication/WalStreamRunner.ts +0 -180
  359. package/src/replication/util.ts +0 -76
  360. package/src/system/CorePowerSyncSystem.ts +0 -64
  361. package/src/util/PgManager.ts +0 -64
  362. package/src/util/migration_lib.ts +0 -79
  363. package/src/util/pgwire_utils.ts +0 -139
  364. package/src/util/populate_test_data.ts +0 -78
  365. package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
  366. package/test/src/large_batch.test.ts +0 -194
  367. package/test/src/pg_test.test.ts +0 -450
  368. package/test/src/schema_changes.test.ts +0 -545
  369. package/test/src/slow_tests.test.ts +0 -338
  370. package/test/src/validation.test.ts +0 -63
  371. package/test/src/wal_stream.test.ts +0 -319
  372. package/test/src/wal_stream_utils.ts +0 -156
@@ -0,0 +1,16 @@
1
+ import * as sync_rules from '@powersync/service-sync-rules';
2
+ import { BucketStorageBatch, SaveOp } from './BucketStorage.js';
3
+ import { SourceTable } from './SourceTable.js';
4
+
5
+ export type EventData = {
6
+ op: SaveOp;
7
+ before?: sync_rules.SqliteRow;
8
+ after?: sync_rules.SqliteRow;
9
+ };
10
+
11
+ export type ReplicationEventPayload = {
12
+ batch: BucketStorageBatch;
13
+ data: EventData;
14
+ event: sync_rules.SqlEventDescriptor;
15
+ table: SourceTable;
16
+ };
@@ -0,0 +1,22 @@
1
+ export interface ColumnDescriptor {
2
+ name: string;
3
+ /**
4
+ * The type of the column ie VARCHAR, INT, etc
5
+ */
6
+ type?: string;
7
+ /**
8
+ * Some data sources have a type id that can be used to identify the type of the column
9
+ */
10
+ typeId?: number;
11
+ }
12
+
13
+ // TODO: This needs to be consolidated with SourceTable into something new.
14
+ export interface SourceEntityDescriptor {
15
+ /**
16
+ * The internal id of the data source structure in the database
17
+ */
18
+ objectId: number | string;
19
+ schema: string;
20
+ name: string;
21
+ replicationColumns: ColumnDescriptor[];
22
+ }
@@ -1,10 +1,8 @@
1
- import { DEFAULT_SCHEMA, DEFAULT_TAG } from '@powersync/service-sync-rules';
2
-
3
- import * as replication from '../replication/replication-index.js';
1
+ import { DEFAULT_TAG } from '@powersync/service-sync-rules';
4
2
  import * as util from '../util/util-index.js';
3
+ import { ColumnDescriptor } from './SourceEntity.js';
5
4
 
6
5
  export class SourceTable {
7
- static readonly DEFAULT_SCHEMA = DEFAULT_SCHEMA;
8
6
  static readonly DEFAULT_TAG = DEFAULT_TAG;
9
7
 
10
8
  /**
@@ -25,14 +23,23 @@ export class SourceTable {
25
23
  */
26
24
  public syncParameters = true;
27
25
 
26
+ /**
27
+ * True if the table is used in sync rules for events.
28
+ *
29
+ * This value is resolved externally, and cached here.
30
+ *
31
+ * Defaults to true for tests.
32
+ */
33
+ public syncEvent = true;
34
+
28
35
  constructor(
29
36
  public readonly id: any,
30
37
  public readonly connectionTag: string,
31
- public readonly relationId: number,
38
+ public readonly objectId: number | string,
32
39
  public readonly schema: string,
33
40
  public readonly table: string,
34
41
 
35
- public readonly replicaIdColumns: replication.ReplicationColumn[],
42
+ public readonly replicaIdColumns: ColumnDescriptor[],
36
43
  public readonly snapshotComplete: boolean
37
44
  ) {}
38
45
 
@@ -55,6 +62,6 @@ export class SourceTable {
55
62
  }
56
63
 
57
64
  get syncAny() {
58
- return this.syncData || this.syncParameters;
65
+ return this.syncData || this.syncParameters || this.syncEvent;
59
66
  }
60
67
  }
@@ -0,0 +1,62 @@
1
+ import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
2
+ import { ResolvedPowerSyncConfig } from '../util/util-index.js';
3
+ import { BucketStorageFactory } from './BucketStorage.js';
4
+ import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
5
+
6
+ export type StorageEngineOptions = {
7
+ configuration: ResolvedPowerSyncConfig;
8
+ };
9
+
10
+ export interface StorageEngineListener extends DisposableListener {
11
+ storageActivated: (storage: BucketStorageFactory) => void;
12
+ }
13
+
14
+ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
15
+ // TODO: This will need to revisited when we actually support multiple storage providers.
16
+ private storageProviders: Map<string, BucketStorageProvider> = new Map();
17
+ private currentActiveStorage: ActiveStorage | null = null;
18
+
19
+ constructor(private options: StorageEngineOptions) {
20
+ super();
21
+ }
22
+
23
+ get activeBucketStorage(): BucketStorageFactory {
24
+ return this.activeStorage.storage;
25
+ }
26
+
27
+ get activeStorage(): ActiveStorage {
28
+ if (!this.currentActiveStorage) {
29
+ throw new Error(`No storage provider has been initialized yet.`);
30
+ }
31
+
32
+ return this.currentActiveStorage;
33
+ }
34
+
35
+ /**
36
+ * Register a provider which generates a {@link BucketStorageFactory}
37
+ * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
38
+ */
39
+ registerProvider(provider: BucketStorageProvider) {
40
+ this.storageProviders.set(provider.type, provider);
41
+ }
42
+
43
+ public async start(): Promise<void> {
44
+ logger.info('Starting Storage Engine...');
45
+ const { configuration } = this.options;
46
+ this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
47
+ resolvedConfig: configuration
48
+ });
49
+ this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
50
+ logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
51
+ logger.info('Successfully started Storage Engine.');
52
+ }
53
+
54
+ /**
55
+ * Shutdown the storage engine, safely shutting down any activated storage providers.
56
+ */
57
+ public async shutDown(): Promise<void> {
58
+ logger.info('Shutting down Storage Engine...');
59
+ await this.currentActiveStorage?.shutDown();
60
+ logger.info('Successfully shut down Storage Engine.');
61
+ }
62
+ }
@@ -0,0 +1,27 @@
1
+ import * as util from '../util/util-index.js';
2
+ import { BucketStorageFactory } from './BucketStorage.js';
3
+
4
+ export interface ActiveStorage {
5
+ storage: BucketStorageFactory;
6
+ shutDown(): Promise<void>;
7
+
8
+ /**
9
+ * Tear down / drop the storage permanently
10
+ */
11
+ tearDown(): Promise<boolean>;
12
+ }
13
+
14
+ export interface GetStorageOptions {
15
+ // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
16
+ resolvedConfig: util.ResolvedPowerSyncConfig;
17
+ }
18
+
19
+ export interface BucketStorageProvider {
20
+ /**
21
+ * The storage type that this provider provides.
22
+ * The type should match the `type` field in the config.
23
+ */
24
+ type: string;
25
+
26
+ getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
27
+ }
@@ -0,0 +1,85 @@
1
+ export enum WriteCheckpointMode {
2
+ /**
3
+ * Raw mappings of `user_id` to `write_checkpoint`s should
4
+ * be supplied for each set of sync rules.
5
+ */
6
+ CUSTOM = 'custom',
7
+ /**
8
+ * Write checkpoints are stored as a mapping of `user_id` plus
9
+ * replication HEAD (lsn in Postgres) to an automatically generated
10
+ * incrementing `write_checkpoint` (stored as`client_id`).
11
+ */
12
+ MANAGED = 'managed'
13
+ }
14
+
15
+ export interface BaseWriteCheckpointIdentifier {
16
+ /**
17
+ * Identifier for User's account.
18
+ */
19
+ user_id: string;
20
+ }
21
+
22
+ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
23
+ /**
24
+ * Sync rules which were active when this checkpoint was created.
25
+ */
26
+ sync_rules_id: number;
27
+ }
28
+
29
+ export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
30
+ /**
31
+ * A supplied incrementing Write Checkpoint number
32
+ */
33
+ checkpoint: bigint;
34
+ }
35
+
36
+ export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
37
+ /**
38
+ * Sync rules which were active when this checkpoint was created.
39
+ */
40
+ sync_rules_id: number;
41
+ }
42
+
43
+ /**
44
+ * Managed Write Checkpoints are a mapping of User ID to replication HEAD
45
+ */
46
+ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
47
+ /**
48
+ * Replication HEAD(s) at the creation of the checkpoint.
49
+ */
50
+ heads: Record<string, string>;
51
+ }
52
+
53
+ export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
54
+
55
+ export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
56
+ export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
57
+
58
+ export interface BaseWriteCheckpointAPI {
59
+ readonly writeCheckpointMode: WriteCheckpointMode;
60
+ setWriteCheckpointMode(mode: WriteCheckpointMode): void;
61
+ createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
62
+ }
63
+
64
+ /**
65
+ * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
66
+ * This storage corresponds with a set of sync rules. These APIs don't require specifying a
67
+ * sync rules id.
68
+ */
69
+ export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
70
+ batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
71
+ createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
72
+ lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
73
+ }
74
+
75
+ /**
76
+ * Write Checkpoint API which is interfaced directly with the storage layer. This requires
77
+ * sync rules identifiers for custom write checkpoints.
78
+ */
79
+ export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
80
+ batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
81
+ createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
82
+ lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
83
+ }
84
+
85
+ export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;
@@ -1,18 +1,25 @@
1
- import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
1
+ import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
2
2
  import * as bson from 'bson';
3
3
  import * as mongo from 'mongodb';
4
4
 
5
+ import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework';
5
6
  import * as util from '../../util/util-index.js';
6
- import * as replication from '../../replication/replication-index.js';
7
- import { container, errors, logger } from '@powersync/lib-services-framework';
8
- import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js';
7
+ import {
8
+ BucketBatchStorageListener,
9
+ BucketStorageBatch,
10
+ FlushedResult,
11
+ mergeToast,
12
+ SaveOptions
13
+ } from '../BucketStorage.js';
9
14
  import { SourceTable } from '../SourceTable.js';
15
+ import { BatchedCustomWriteCheckpointOptions, CustomWriteCheckpointOptions } from '../WriteCheckpointAPI.js';
10
16
  import { PowerSyncMongo } from './db.js';
11
- import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js';
17
+ import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
12
18
  import { MongoIdSequence } from './MongoIdSequence.js';
19
+ import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
13
20
  import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
14
21
  import { PersistedBatch } from './PersistedBatch.js';
15
- import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, serializeLookup } from './util.js';
22
+ import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
16
23
 
17
24
  /**
18
25
  * 15MB
@@ -25,7 +32,18 @@ const MAX_ROW_SIZE = 15 * 1024 * 1024;
25
32
  //
26
33
  // In the future, we can investigate allowing multiple replication streams operating independently.
27
34
  const replicationMutex = new util.Mutex();
28
- export class MongoBucketBatch implements BucketStorageBatch {
35
+
36
+ export interface MongoBucketBatchOptions {
37
+ db: PowerSyncMongo;
38
+ syncRules: SqlSyncRules;
39
+ groupId: number;
40
+ slotName: string;
41
+ lastCheckpointLsn: string | null;
42
+ noCheckpointBeforeLsn: string;
43
+ storeCurrentData: boolean;
44
+ }
45
+
46
+ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListener> implements BucketStorageBatch {
29
47
  private readonly client: mongo.MongoClient;
30
48
  public readonly db: PowerSyncMongo;
31
49
  public readonly session: mongo.ClientSession;
@@ -34,8 +52,10 @@ export class MongoBucketBatch implements BucketStorageBatch {
34
52
  private readonly group_id: number;
35
53
 
36
54
  private readonly slot_name: string;
55
+ private readonly storeCurrentData: boolean;
37
56
 
38
57
  private batch: OperationBatch | null = null;
58
+ private write_checkpoint_batch: CustomWriteCheckpointOptions[] = [];
39
59
 
40
60
  /**
41
61
  * Last LSN received associated with a checkpoint.
@@ -55,22 +75,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
55
75
  */
56
76
  public last_flushed_op: bigint | null = null;
57
77
 
58
- constructor(
59
- db: PowerSyncMongo,
60
- sync_rules: SqlSyncRules,
61
- group_id: number,
62
- slot_name: string,
63
- last_checkpoint_lsn: string | null,
64
- no_checkpoint_before_lsn: string | null
65
- ) {
66
- this.db = db;
67
- this.client = db.client;
68
- this.sync_rules = sync_rules;
69
- this.group_id = group_id;
70
- this.slot_name = slot_name;
78
+ constructor(options: MongoBucketBatchOptions) {
79
+ super();
80
+ this.client = options.db.client;
81
+ this.db = options.db;
82
+ this.group_id = options.groupId;
83
+ this.last_checkpoint_lsn = options.lastCheckpointLsn;
84
+ this.no_checkpoint_before_lsn = options.noCheckpointBeforeLsn;
71
85
  this.session = this.client.startSession();
72
- this.last_checkpoint_lsn = last_checkpoint_lsn;
73
- this.no_checkpoint_before_lsn = no_checkpoint_before_lsn ?? replication.ZERO_LSN;
86
+ this.slot_name = options.slotName;
87
+ this.sync_rules = options.syncRules;
88
+ this.storeCurrentData = options.storeCurrentData;
89
+ this.batch = new OperationBatch();
90
+ }
91
+
92
+ addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void {
93
+ this.write_checkpoint_batch.push({
94
+ ...checkpoint,
95
+ sync_rules_id: this.group_id
96
+ });
97
+ }
98
+
99
+ get lastCheckpointLsn() {
100
+ return this.last_checkpoint_lsn;
74
101
  }
75
102
 
76
103
  async flush(): Promise<FlushedResult | null> {
@@ -83,6 +110,8 @@ export class MongoBucketBatch implements BucketStorageBatch {
83
110
  result = r;
84
111
  }
85
112
  }
113
+ await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch);
114
+ this.write_checkpoint_batch = [];
86
115
  return result;
87
116
  }
88
117
 
@@ -118,38 +147,44 @@ export class MongoBucketBatch implements BucketStorageBatch {
118
147
  batch: OperationBatch,
119
148
  op_seq: MongoIdSequence
120
149
  ): Promise<OperationBatch | null> {
121
- // 1. Find sizes of current_data documents, to assist in intelligent batching without
122
- // exceeding memory limits.
123
- //
124
- // A previous attempt tried to do batching by the results of the current_data query
125
- // (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes
126
- // the order of processing, which then becomes really tricky to manage.
127
- // This now takes 2+ queries, but doesn't have any issues with order of operations.
128
- const sizeLookups: SourceKey[] = batch.batch.map((r) => {
129
- return { g: this.group_id, t: r.record.sourceTable.id, k: r.beforeId };
130
- });
150
+ let sizes: Map<string, number> | undefined = undefined;
151
+ if (this.storeCurrentData) {
152
+ // We skip this step if we don't store current_data, since the sizes will
153
+ // always be small in that case.
154
+
155
+ // Find sizes of current_data documents, to assist in intelligent batching without
156
+ // exceeding memory limits.
157
+ //
158
+ // A previous attempt tried to do batching by the results of the current_data query
159
+ // (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes
160
+ // the order of processing, which then becomes really tricky to manage.
161
+ // This now takes 2+ queries, but doesn't have any issues with order of operations.
162
+ const sizeLookups: SourceKey[] = batch.batch.map((r) => {
163
+ return { g: this.group_id, t: r.record.sourceTable.id, k: r.beforeId };
164
+ });
131
165
 
132
- const sizes = new Map<string, number>();
166
+ sizes = new Map<string, number>();
133
167
 
134
- const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = this.db.current_data.aggregate(
135
- [
136
- {
137
- $match: {
138
- _id: { $in: sizeLookups }
139
- }
140
- },
141
- {
142
- $project: {
143
- _id: 1,
144
- size: { $bsonSize: '$$ROOT' }
168
+ const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = this.db.current_data.aggregate(
169
+ [
170
+ {
171
+ $match: {
172
+ _id: { $in: sizeLookups }
173
+ }
174
+ },
175
+ {
176
+ $project: {
177
+ _id: 1,
178
+ size: { $bsonSize: '$$ROOT' }
179
+ }
145
180
  }
146
- }
147
- ],
148
- { session }
149
- );
150
- for await (let doc of sizeCursor.stream()) {
151
- const key = cacheKey(doc._id.t, doc._id.k);
152
- sizes.set(key, doc.size);
181
+ ],
182
+ { session }
183
+ );
184
+ for await (let doc of sizeCursor.stream()) {
185
+ const key = cacheKey(doc._id.t, doc._id.k);
186
+ sizes.set(key, doc.size);
187
+ }
153
188
  }
154
189
 
155
190
  // If set, we need to start a new transaction with this batch.
@@ -157,6 +192,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
157
192
  let transactionSize = 0;
158
193
 
159
194
  // Now batch according to the sizes
195
+ // This is a single batch if storeCurrentData == false
160
196
  for await (let b of batch.batched(sizes)) {
161
197
  if (resumeBatch) {
162
198
  for (let op of b) {
@@ -194,7 +230,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
194
230
  if (nextData != null) {
195
231
  // Update our current_data and size cache
196
232
  current_data_lookup.set(op.internalAfterKey!, nextData);
197
- sizes.set(op.internalAfterKey!, nextData.data.length());
233
+ sizes?.set(op.internalAfterKey!, nextData.data.length());
198
234
  }
199
235
 
200
236
  if (persistedBatch!.shouldFlushTransaction()) {
@@ -244,14 +280,18 @@ export class MongoBucketBatch implements BucketStorageBatch {
244
280
  existing_buckets = [];
245
281
  existing_lookups = [];
246
282
  // Log to help with debugging if there was a consistency issue
247
- logger.warn(
248
- `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
249
- );
283
+ if (this.storeCurrentData) {
284
+ logger.warn(
285
+ `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
286
+ );
287
+ }
250
288
  } else {
251
- const data = bson.deserialize((result.data as mongo.Binary).buffer, BSON_DESERIALIZE_OPTIONS) as SqliteRow;
252
289
  existing_buckets = result.buckets;
253
290
  existing_lookups = result.lookups;
254
- after = mergeToast(after!, data);
291
+ if (this.storeCurrentData) {
292
+ const data = bson.deserialize((result.data as mongo.Binary).buffer, BSON_DESERIALIZE_OPTIONS) as SqliteRow;
293
+ after = mergeToast(after!, data);
294
+ }
255
295
  }
256
296
  } else if (record.tag == 'delete') {
257
297
  const result = current_data;
@@ -260,9 +300,11 @@ export class MongoBucketBatch implements BucketStorageBatch {
260
300
  existing_buckets = [];
261
301
  existing_lookups = [];
262
302
  // Log to help with debugging if there was a consistency issue
263
- logger.warn(
264
- `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
265
- );
303
+ if (this.storeCurrentData) {
304
+ logger.warn(
305
+ `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
306
+ );
307
+ }
266
308
  } else {
267
309
  existing_buckets = result.buckets;
268
310
  existing_lookups = result.lookups;
@@ -270,7 +312,9 @@ export class MongoBucketBatch implements BucketStorageBatch {
270
312
  }
271
313
 
272
314
  let afterData: bson.Binary | undefined;
273
- if (afterId) {
315
+ if (afterId != null && !this.storeCurrentData) {
316
+ afterData = new bson.Binary(bson.serialize({}));
317
+ } else if (afterId != null) {
274
318
  try {
275
319
  // This will fail immediately if the record is > 16MB.
276
320
  afterData = new bson.Binary(bson.serialize(after!));
@@ -301,7 +345,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
301
345
  }
302
346
 
303
347
  // 2. Save bucket data
304
- if (beforeId != null && (afterId == null || !beforeId.equals(afterId))) {
348
+ if (beforeId != null && (afterId == null || !replicaIdEquals(beforeId, afterId))) {
305
349
  // Source ID updated
306
350
  if (sourceTable.syncData) {
307
351
  // Delete old record
@@ -431,7 +475,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
431
475
  };
432
476
  }
433
477
 
434
- if (afterId == null || !beforeId.equals(afterId)) {
478
+ if (afterId == null || !replicaIdEquals(beforeId, afterId)) {
435
479
  // Either a delete (afterId == null), or replaced the old replication id
436
480
  batch.deleteCurrentData(before_key);
437
481
  }
@@ -528,14 +572,15 @@ export class MongoBucketBatch implements BucketStorageBatch {
528
572
  });
529
573
  }
530
574
 
531
- async abort() {
575
+ async [Symbol.asyncDispose]() {
532
576
  await this.session.endSession();
577
+ super[Symbol.dispose]();
533
578
  }
534
579
 
535
580
  async commit(lsn: string): Promise<boolean> {
536
581
  await this.flush();
537
582
 
538
- if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) {
583
+ if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
539
584
  // When re-applying transactions, don't create a new checkpoint until
540
585
  // we are past the last transaction.
541
586
  logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);
@@ -546,26 +591,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
546
591
  return false;
547
592
  }
548
593
 
594
+ const now = new Date();
595
+ const update: Partial<SyncRuleDocument> = {
596
+ last_checkpoint_lsn: lsn,
597
+ last_checkpoint_ts: now,
598
+ last_keepalive_ts: now,
599
+ snapshot_done: true,
600
+ last_fatal_error: null
601
+ };
602
+
549
603
  if (this.persisted_op != null) {
550
- const now = new Date();
551
- await this.db.sync_rules.updateOne(
552
- {
553
- _id: this.group_id
554
- },
555
- {
556
- $set: {
557
- last_checkpoint: this.persisted_op,
558
- last_checkpoint_lsn: lsn,
559
- last_checkpoint_ts: now,
560
- last_keepalive_ts: now,
561
- snapshot_done: true,
562
- last_fatal_error: null
563
- }
564
- },
565
- { session: this.session }
566
- );
567
- this.persisted_op = null;
604
+ update.last_checkpoint = this.persisted_op;
568
605
  }
606
+
607
+ await this.db.sync_rules.updateOne(
608
+ {
609
+ _id: this.group_id
610
+ },
611
+ {
612
+ $set: update
613
+ },
614
+ { session: this.session }
615
+ );
616
+ this.persisted_op = null;
569
617
  this.last_checkpoint_lsn = lsn;
570
618
  return true;
571
619
  }
@@ -606,6 +654,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
606
654
  }
607
655
 
608
656
  async save(record: SaveOptions): Promise<FlushedResult | null> {
657
+ const { after, before, sourceTable, tag } = record;
658
+ for (const event of this.getTableEvents(sourceTable)) {
659
+ this.iterateListeners((cb) =>
660
+ cb.replicationEvent?.({
661
+ batch: this,
662
+ table: sourceTable,
663
+ data: {
664
+ op: tag,
665
+ after: after && util.isCompleteRow(after) ? after : undefined,
666
+ before: before && util.isCompleteRow(before) ? before : undefined
667
+ },
668
+ event
669
+ })
670
+ );
671
+ }
672
+
673
+ /**
674
+ * Return if the table is just an event table
675
+ */
676
+ if (!sourceTable.syncData && !sourceTable.syncParameters) {
677
+ return null;
678
+ }
679
+
609
680
  logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
610
681
 
611
682
  this.batch ??= new OperationBatch();
@@ -743,7 +814,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
743
814
  const copy = new SourceTable(
744
815
  table.id,
745
816
  table.connectionTag,
746
- table.relationId,
817
+ table.objectId,
747
818
  table.schema,
748
819
  table.table,
749
820
  table.replicaIdColumns,
@@ -754,6 +825,15 @@ export class MongoBucketBatch implements BucketStorageBatch {
754
825
  return copy;
755
826
  });
756
827
  }
828
+
829
+ /**
830
+ * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
831
+ */
832
+ protected getTableEvents(table: SourceTable): SqlEventDescriptor[] {
833
+ return this.sync_rules.event_descriptors.filter((evt) =>
834
+ [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
835
+ );
836
+ }
757
837
  }
758
838
 
759
839
  export function currentBucketKey(b: CurrentBucket) {