@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +177 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +71 -18
  6. package/dist/storage/MongoBucketStorage.js +516 -202
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
  9. package/dist/storage/implementation/MongoBucketBatch.js +140 -66
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
  15. package/dist/storage/implementation/MongoCompactor.js +87 -481
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
  21. package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +41 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
  47. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  48. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  49. package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
  50. package/dist/storage/implementation/common/PersistedBatch.js +63 -10
  51. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +4 -1
  58. package/dist/storage/implementation/db.js +31 -16
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +55 -13
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  67. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  68. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
  69. package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
  70. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  72. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  73. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  74. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
  75. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
  76. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  77. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  78. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  79. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  80. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  81. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  82. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  83. package/dist/storage/implementation/v1/models.d.ts +10 -1
  84. package/dist/storage/implementation/v1/models.js +1 -0
  85. package/dist/storage/implementation/v1/models.js.map +1 -1
  86. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  87. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  88. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  89. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  90. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  91. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
  93. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
  94. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  96. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  97. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
  99. package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
  100. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
  102. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
  103. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
  105. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
  106. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  107. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
  108. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
  109. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  110. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  111. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  112. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  113. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
  114. package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
  115. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  116. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  117. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  118. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  119. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
  120. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
  121. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  122. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  123. package/dist/storage/implementation/v3/bucket-format.js +79 -0
  124. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  125. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  126. package/dist/storage/implementation/v3/chunking.js +30 -0
  127. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  128. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  129. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  130. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  131. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  132. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  133. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  134. package/dist/storage/implementation/v3/models.d.ts +187 -23
  135. package/dist/storage/implementation/v3/models.js +11 -28
  136. package/dist/storage/implementation/v3/models.js.map +1 -1
  137. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
  138. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
  139. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  140. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
  141. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  142. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  143. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
  144. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
  145. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  146. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
  147. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
  148. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  149. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
  150. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
  151. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  152. package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
  153. package/dist/storage/implementation/v3/source-table-utils.js +206 -0
  154. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  155. package/dist/storage/storage-index.d.ts +5 -4
  156. package/dist/storage/storage-index.js +5 -4
  157. package/dist/storage/storage-index.js.map +1 -1
  158. package/dist/types/types.d.ts +71 -3
  159. package/dist/types/types.js +58 -2
  160. package/dist/types/types.js.map +1 -1
  161. package/dist/utils/test-utils.d.ts +1 -2
  162. package/dist/utils/test-utils.js +7 -1
  163. package/dist/utils/test-utils.js.map +1 -1
  164. package/dist/utils/util.d.ts +9 -2
  165. package/dist/utils/util.js +131 -5
  166. package/dist/utils/util.js.map +1 -1
  167. package/package.json +13 -19
  168. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  169. package/src/storage/MongoBucketStorage.ts +728 -242
  170. package/src/storage/implementation/MongoBucketBatch.ts +196 -91
  171. package/src/storage/implementation/MongoChecksums.ts +93 -153
  172. package/src/storage/implementation/MongoCompactor.ts +106 -631
  173. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  174. package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
  175. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
  176. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  177. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
  178. package/src/storage/implementation/MongoStorageProvider.ts +48 -4
  179. package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
  180. package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
  181. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
  182. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  183. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
  184. package/src/storage/implementation/common/PersistedBatch.ts +83 -15
  185. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  186. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  187. package/src/storage/implementation/db.ts +40 -22
  188. package/src/storage/implementation/models.ts +58 -14
  189. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
  190. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  191. package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
  192. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  193. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
  194. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  195. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  196. package/src/storage/implementation/v1/models.ts +12 -1
  197. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  198. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  199. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
  200. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  201. package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
  202. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
  203. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
  204. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
  205. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  206. package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
  207. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  208. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
  209. package/src/storage/implementation/v3/bucket-format.ts +99 -0
  210. package/src/storage/implementation/v3/chunking.ts +40 -0
  211. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  212. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  213. package/src/storage/implementation/v3/models.ts +228 -58
  214. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
  215. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
  216. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
  217. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
  218. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
  219. package/src/storage/implementation/v3/source-table-utils.ts +398 -0
  220. package/src/storage/storage-index.ts +14 -4
  221. package/src/types/types.ts +76 -2
  222. package/src/utils/test-utils.ts +8 -3
  223. package/src/utils/util.ts +188 -7
  224. package/test/src/__snapshots__/storage.test.ts.snap +4 -54
  225. package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
  226. package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
  227. package/test/src/compact-utils.test.ts +226 -0
  228. package/test/src/helpers/MemoryObjectStorage.ts +61 -0
  229. package/test/src/helpers/s3TestFactory.ts +69 -0
  230. package/test/src/object_storage_lifecycle.test.ts +59 -0
  231. package/test/src/object_storage_usage.test.ts +255 -0
  232. package/test/src/parameter_compacting_v1.test.ts +120 -0
  233. package/test/src/parameter_compacting_v3.test.ts +192 -0
  234. package/test/src/parameter_compaction_fence.test.ts +276 -0
  235. package/test/src/setup.ts +6 -1
  236. package/test/src/storage.test.ts +478 -13
  237. package/test/src/storage_compacting.test.ts +2516 -107
  238. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  239. package/test/src/storage_s3_checksums.test.ts +190 -0
  240. package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
  241. package/test/src/storage_s3_reading.test.ts +768 -0
  242. package/test/src/storage_s3_writing.test.ts +136 -0
  243. package/test/src/storage_sync.test.ts +1507 -76
  244. package/test/src/storeCurrentData.test.ts +227 -0
  245. package/test/src/util.ts +3 -2
  246. package/tsconfig.json +1 -4
  247. package/tsconfig.tsbuildinfo +1 -1
  248. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  249. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  250. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  251. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  252. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  253. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  254. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  255. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  256. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  257. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  258. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  259. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  260. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  261. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  262. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  263. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  264. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  265. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  266. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  267. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  268. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  269. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  270. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  271. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  272. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  273. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  274. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  275. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,13 +1,25 @@
1
- import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
1
+ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
2
+ import {
3
+ deserializeParameterLookup,
4
+ JwtPayload,
5
+ SingleSyncConfigBucketDefinitionMapping,
6
+ storage,
7
+ updateSyncRulesFromYaml
8
+ } from '@powersync/service-core';
2
9
  import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
3
- import { DEFAULT_HYDRATION_STATE, nodeSqlite, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
10
+ import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
4
11
  import * as bson from 'bson';
5
- import * as sqlite from 'node:sqlite';
6
12
  import { describe, expect, test } from 'vitest';
7
13
  import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
14
+ import { MongoParsedSyncConfigSet } from '../../src/storage/implementation/MongoParsedSyncConfigSet.js';
15
+ import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementation/MongoPersistedSyncConfigContent.js';
16
+ import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
8
17
  import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
18
+ import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
19
+ import { loadBucketDataDocumentV1, serializeBucketDataV1 } from '../../src/storage/implementation/v1/models.js';
9
20
  import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
10
21
  import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
22
+ import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
11
23
  import {
12
24
  CurrentBucketV3,
13
25
  ReplicationStreamDocumentV3,
@@ -53,17 +65,115 @@ function objectIdGenerator(id: string) {
53
65
  };
54
66
  }
55
67
 
56
- function hydratedRulesFor(yaml: string) {
68
+ /**
69
+ * Build a standalone parsed sync config set for the given yaml, with its own mapping.
70
+ *
71
+ * This keeps the sync rules and the bucket definition mapping paired, as required by
72
+ * `ResolveTablesOptions.parsedSyncConfig`.
73
+ */
74
+ function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
57
75
  const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
58
76
  expect(parsed.errors).toEqual([]);
59
- return parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) });
77
+ return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
78
+ {
79
+ syncConfigId: new bson.ObjectId().toHexString(),
80
+ syncConfig: parsed,
81
+ mapping: SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(parsed)
82
+ }
83
+ ]);
84
+ }
85
+
86
+ async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory, replicationStreamId: number) {
87
+ const mongoFactory = factory as MongoBucketStorage;
88
+ const doc = (await mongoFactory.db.sync_rules.findOne({
89
+ _id: replicationStreamId
90
+ })) as ReplicationStreamDocumentV3 | null;
91
+ if (doc == null) {
92
+ return [];
93
+ }
94
+
95
+ const db = mongoFactory.db.versioned(getMongoStorageConfig(doc.storage_version)) as VersionedPowerSyncMongoV3;
96
+ const syncConfigDocs = await db.syncConfigDefinitions
97
+ .find({
98
+ _id: { $in: doc.sync_configs.map((config) => config._id) }
99
+ })
100
+ .toArray();
101
+
102
+ return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
60
103
  }
61
104
 
105
+ /**
106
+ * Get a MongoDB V3 storage bucket definition id, from a bucket name.
107
+ *
108
+ * This relies on internals of bucket naming, but helps for simplifying tests.
109
+ */
110
+ function bucketDefinitionId(bucket: string) {
111
+ const match = bucket.match(/^\w+\|\w+\.\w+\.(\w+)\[/);
112
+ if (match == null) {
113
+ throw new Error(`Expected versioned bucket name, got ${bucket}`);
114
+ }
115
+ return parseInt(match[1], 16);
116
+ }
117
+
118
+ test('V1 bucket data retains an optional persisted subkey', () => {
119
+ const sourceTable = new bson.ObjectId();
120
+ const document = serializeBucketDataV1({
121
+ bucketKey: {
122
+ replicationStreamId: 1,
123
+ definitionId: '0',
124
+ bucket: 'global[]'
125
+ },
126
+ o: 1n,
127
+ op: 'PUT',
128
+ source_table: sourceTable,
129
+ source_key: 'source-key',
130
+ subkey: 'persisted-subkey',
131
+ table: 'items',
132
+ row_id: 'item-1',
133
+ checksum: 1n,
134
+ data: '{"id":"item-1"}'
135
+ });
136
+
137
+ expect(loadBucketDataDocumentV1(document).subkey).toBe('persisted-subkey');
138
+
139
+ const { subkey: _subkey, ...legacyDocument } = document;
140
+ expect(loadBucketDataDocumentV1(legacyDocument).subkey).toBeUndefined();
141
+ });
142
+
62
143
  function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
63
144
  register.registerSyncTests(storageConfig.factory, {
64
145
  storageVersion,
65
146
  tableIdStrings: storageConfig.tableIdStrings
66
147
  });
148
+
149
+ test('updates source metadata on an existing resolved table', async () => {
150
+ await using factory = await storageConfig.factory();
151
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
152
+ const bucketStorage = factory.getInstance(syncRules);
153
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
154
+ const source = sourceDescriptor('test');
155
+
156
+ const initial = await writer.resolveTables({ connection_id: 1, source });
157
+ expect(initial.tables[0].sourceMetadata).toBeNull();
158
+
159
+ const sourceMetadata = { captureTableObjectId: 42 };
160
+ const updated = await writer.resolveTables({
161
+ connection_id: 1,
162
+ source,
163
+ reconcileSourceTables: ({ candidates }) => ({
164
+ compatibleTables: candidates.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
165
+ incompatibleTables: [],
166
+ newTableValues: { sourceMetadata }
167
+ })
168
+ });
169
+
170
+ expect(updated.tables.map((table) => table.id.toString())).toEqual(
171
+ initial.tables.map((table) => table.id.toString())
172
+ );
173
+ expect(updated.tables[0].sourceMetadata).toEqual(sourceMetadata);
174
+ expect((await writer.getSourceTableStatus(updated.tables[0]))?.sourceMetadata).toEqual(sourceMetadata);
175
+ });
176
+
67
177
  // The split of returned results can vary depending on storage drivers
68
178
  test('large batch (2)', async () => {
69
179
  // Test syncing a batch of data that is small in count,
@@ -82,6 +192,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
82
192
  )
83
193
  );
84
194
  const bucketStorage = factory.getInstance(syncRules);
195
+ const syncRulesContent = syncRules.syncConfigContent[0];
85
196
 
86
197
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
87
198
 
@@ -135,8 +246,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
135
246
  const checkpoint = flushResult!.flushed_op;
136
247
 
137
248
  const options: storage.BucketDataBatchOptions = {};
138
- const batch1 = await test_utils.fromAsync(
139
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)], options)
249
+ const batch1 = await test_utils.getBatchArray(
250
+ bucketStorage.getBucketDataBatch(
251
+ test_utils.testCheckpoint(checkpoint),
252
+ [bucketRequest(syncRulesContent, 'global[]', 0n)],
253
+ options
254
+ )
140
255
  );
141
256
  expect(test_utils.getBatchData(batch1)).toEqual([
142
257
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -148,10 +263,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
148
263
  next_after: '2'
149
264
  });
150
265
 
151
- const batch2 = await test_utils.fromAsync(
266
+ const batch2 = await test_utils.getBatchArray(
152
267
  bucketStorage.getBucketDataBatch(
153
- checkpoint,
154
- [bucketRequest(syncRules, 'global[]', batch1[0].chunkData.next_after)],
268
+ test_utils.testCheckpoint(checkpoint),
269
+ [bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
155
270
  options
156
271
  )
157
272
  );
@@ -164,10 +279,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
164
279
  next_after: '3'
165
280
  });
166
281
 
167
- const batch3 = await test_utils.fromAsync(
282
+ const batch3 = await test_utils.getBatchArray(
168
283
  bucketStorage.getBucketDataBatch(
169
- checkpoint,
170
- [bucketRequest(syncRules, 'global[]', batch2[0].chunkData.next_after)],
284
+ test_utils.testCheckpoint(checkpoint),
285
+ [bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
171
286
  options
172
287
  )
173
288
  );
@@ -189,7 +304,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
189
304
  await Promise.all(
190
305
  (
191
306
  await mongoFactory.db.db
192
- .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
307
+ .listCollections(
308
+ { name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) },
309
+ { nameOnly: true }
310
+ )
193
311
  .toArray()
194
312
  ).map((collection: { name: string }) =>
195
313
  mongoFactory.db.db
@@ -388,8 +506,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
388
506
  }
389
507
  );
390
508
 
391
- test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions and removals', async () => {
392
- // Tests the behavior of resolveTables when bucket data sources and parameter index creators are added or removed.
509
+ test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions', async () => {
510
+ // Tests the behavior of resolveTables when bucket data sources and parameter index creators are added.
393
511
  // These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
394
512
  // This just tests the specific resolveTables behavior.
395
513
 
@@ -434,20 +552,20 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
434
552
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
435
553
  const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
436
554
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
437
- const fullRules = hydratedRulesFor(fullRulesYaml);
438
- const dataOnlyRules = hydratedRulesFor(dataOnlyRulesYaml);
439
- const parameterOnlyRules = hydratedRulesFor(parameterOnlyRulesYaml);
440
- const eventOnlyRules = hydratedRulesFor(eventOnlyRulesYaml);
555
+ const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
556
+ const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
557
+ const parameterOnlyRules = parsedSyncConfigSetFor(parameterOnlyRulesYaml, storageVersion);
558
+ const eventOnlyRules = parsedSyncConfigSetFor(eventOnlyRulesYaml, storageVersion);
441
559
  const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
442
560
  const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
443
561
  const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
444
- const removedDataTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
562
+ const addedEventTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
445
563
 
446
564
  const dataOnly = await writer.resolveTables({
447
565
  connection_id: 1,
448
566
  source,
449
567
  idGenerator: () => dataOnlyTableId,
450
- syncRules: dataOnlyRules
568
+ parsedSyncConfig: dataOnlyRules
451
569
  });
452
570
  expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
453
571
  expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
@@ -458,7 +576,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
458
576
  connection_id: 1,
459
577
  source,
460
578
  idGenerator: () => addedParameterTableId,
461
- syncRules: fullRules
579
+ parsedSyncConfig: fullRules
462
580
  });
463
581
  // Adding a definition always creates a new SourceTable
464
582
  expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
@@ -472,41 +590,40 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
472
590
  idGenerator: () => {
473
591
  throw new Error('data-only resolve should reuse existing v3 source table');
474
592
  },
475
- syncRules: dataOnlyRules
593
+ parsedSyncConfig: dataOnlyRules
476
594
  });
477
595
  expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
478
- // Now this sourceTable is unused & dropped
479
- expect(removedParameter.dropTables.map((table) => table.id)).toEqual([addedParameterTableId]);
596
+ // Same-identity tables that no longer cover desired memberships are left in place.
597
+ expect(removedParameter.dropTables.map((table) => table.id)).toEqual([]);
480
598
  expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
481
599
  expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
482
- await writer.drop(removedParameter.dropTables);
483
600
 
484
601
  const removedData = await writer.resolveTables({
485
602
  connection_id: 1,
486
603
  source,
487
- idGenerator: () => removedDataTableId,
488
- syncRules: parameterOnlyRules
604
+ idGenerator: () => {
605
+ throw new Error('parameter-only resolve should reuse existing v3 source table');
606
+ },
607
+ parsedSyncConfig: parameterOnlyRules
489
608
  });
490
609
 
491
- // This goes from dataOnlyRules -> parameterOnlyRules, which adds one definition and removes another.
492
- // This generates a new SourceTable again, and removes all others.
493
- expect(removedData.tables.map((table) => table.id)).toEqual([removedDataTableId]);
494
- expect(removedData.dropTables.map((table) => table.id)).toEqual([dataOnlyTableId]);
610
+ // This goes from dataOnlyRules -> parameterOnlyRules, which removes one definition but
611
+ // reuses the existing parameter-only source table.
612
+ expect(removedData.tables.map((table) => table.id)).toEqual([addedParameterTableId]);
613
+ expect(removedData.dropTables.map((table) => table.id)).toEqual([]);
495
614
  expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
496
615
  expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
497
- await writer.drop(removedData.dropTables);
498
616
 
499
617
  const eventOnly = await writer.resolveTables({
500
618
  connection_id: 1,
501
619
  source,
502
- idGenerator: () => {
503
- throw new Error('resolve should reuse existing v3 source table');
504
- },
505
- syncRules: eventOnlyRules
620
+ idGenerator: () => addedEventTableId,
621
+ parsedSyncConfig: eventOnlyRules
506
622
  });
507
623
 
508
- // Event-only table can re-use any existing table.
509
- expect(eventOnly.tables.map((table) => table.id)).toEqual([removedDataTableId]);
624
+ // A newly-added event gets its own source table so that its existing rows are snapshotted.
625
+ expect(eventOnly.tables).toHaveLength(1);
626
+ expect(eventOnly.tables[0].id).toEqual(addedEventTableId);
510
627
  expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
511
628
  expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
512
629
  expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
@@ -515,6 +632,234 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
515
632
  expect(eventOnly.tables[0].syncEvent).toBe(true);
516
633
  });
517
634
 
635
+ test.runIf(storageVersion >= 3)('resolveTables assigns an event definition to one split source table', async () => {
636
+ // When memberships are split over multiple SourceTables for the same ref, a row change
637
+ // is saved once per table. Each event-definition id must belong to only one of those tables,
638
+ // otherwise that definition would be evaluated more than once for the same row change.
639
+ // Other tables may independently own and evaluate different event definitions.
640
+ const dataOnlyEventYaml = `
641
+ bucket_definitions:
642
+ by_owner:
643
+ parameters:
644
+ - SELECT token_parameters.owner_id as owner_id
645
+ data:
646
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
647
+
648
+ event_definitions:
649
+ write_checkpoints:
650
+ payloads:
651
+ - SELECT id, owner_id FROM memberships
652
+ `;
653
+ const fullEventYaml = `
654
+ bucket_definitions:
655
+ by_owner:
656
+ parameters:
657
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
658
+ data:
659
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
660
+
661
+ event_definitions:
662
+ write_checkpoints:
663
+ payloads:
664
+ - SELECT id, owner_id FROM memberships
665
+ `;
666
+
667
+ await using factory = await storageConfig.factory();
668
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
669
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
670
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
671
+ const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
672
+ const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
673
+ const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
674
+
675
+ const initial = await writer.resolveTables({
676
+ connection_id: 1,
677
+ source,
678
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
679
+ parsedSyncConfig: dataOnlyRules
680
+ });
681
+ expect(initial.tables).toHaveLength(1);
682
+ expect(initial.tables[0].syncEvent).toBe(true);
683
+
684
+ // Adding the table-based parameter lookup creates a second SourceTable for the same ref.
685
+ const split = await writer.resolveTables({
686
+ connection_id: 1,
687
+ source,
688
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
689
+ parsedSyncConfig: fullRules
690
+ });
691
+ expect(split.tables).toHaveLength(2);
692
+ // Both tables match the event by ref, but this event id belongs to only one of them.
693
+ const eventOwners = split.tables.filter((table) => table.syncEvent);
694
+ expect(eventOwners).toHaveLength(1);
695
+
696
+ // getSourceTableStatus rehydrates the persisted event memberships rather than recomputing
697
+ // them from the ref, so refreshing the other table does not assign this event to it.
698
+ const nonOwner = split.tables.find((table) => !table.syncEvent)!;
699
+ const refreshed = await writer.getSourceTableStatus(nonOwner);
700
+ expect(refreshed!.syncEvent).toBe(false);
701
+ });
702
+
703
+ test.runIf(storageVersion >= 3)(
704
+ 'reuses an unchanged event definition without resnapshotting or duplicate evaluation',
705
+ async () => {
706
+ const firstYaml = `
707
+ config:
708
+ edition: 3
709
+
710
+ streams:
711
+ by_owner:
712
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
713
+
714
+ event_definitions:
715
+ write_checkpoints:
716
+ payloads:
717
+ - SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0
718
+ `;
719
+ const secondYaml = `
720
+ config:
721
+ edition: 3
722
+
723
+ streams:
724
+ by_owner:
725
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
726
+ by_project:
727
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
728
+
729
+ event_definitions:
730
+ write_checkpoints:
731
+ payloads:
732
+ - select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0
733
+ `;
734
+
735
+ await using factory = await storageConfig.factory();
736
+ const emittedEvents: { name: string; eventId: string | undefined }[] = [];
737
+ const disposeListener = factory.registerListener({
738
+ replicationEvent: ({ event, event_id }) => emittedEvents.push({ name: event.name, eventId: event_id })
739
+ });
740
+ try {
741
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion }));
742
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
743
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
744
+ const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
745
+ const firstResolved = await firstWriter.resolveTables({
746
+ connection_id: 1,
747
+ source,
748
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38351')
749
+ });
750
+ const eventId = [...firstResolved.tables[0].eventDefinitionIds!][0];
751
+ expect(eventId).toBeDefined();
752
+ await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1');
753
+ await firstWriter.markAllSnapshotDone('1/1');
754
+ await firstWriter.commit('1/1');
755
+
756
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(secondYaml, { storageVersion }));
757
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
758
+ const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage;
759
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
760
+ const resolved = await secondWriter.resolveTables({
761
+ connection_id: 1,
762
+ source,
763
+ idGenerator: () => {
764
+ throw new Error('unchanged event definition should reuse the snapshotted source table');
765
+ }
766
+ });
767
+
768
+ expect(resolved.tables).toHaveLength(1);
769
+ expect(resolved.tables[0].id).toEqual(firstResolved.tables[0].id);
770
+ expect(resolved.tables[0].snapshotComplete).toBe(true);
771
+ expect([...resolved.tables[0].eventDefinitionIds!]).toEqual([eventId]);
772
+
773
+ emittedEvents.length = 0;
774
+ await secondWriter.save({
775
+ sourceTable: resolved.tables[0],
776
+ tag: storage.SaveOperationTag.INSERT,
777
+ after: { id: 'checkpoint-1', user_id: 'user-1', checkpoint: 1n, active: 1 },
778
+ afterReplicaId: test_utils.rid('checkpoint-1')
779
+ });
780
+ expect(emittedEvents).toEqual([{ name: 'write_checkpoints', eventId }]);
781
+ } finally {
782
+ disposeListener();
783
+ }
784
+ }
785
+ );
786
+
787
+ test.runIf(storageVersion >= 3)('creates snapshot work for a changed event definition', async () => {
788
+ const yaml = (active: boolean) => `
789
+ config:
790
+ edition: 3
791
+
792
+ streams:
793
+ by_owner:
794
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
795
+
796
+ event_definitions:
797
+ write_checkpoints:
798
+ payloads:
799
+ - SELECT user_id, checkpoint FROM checkpoints WHERE active = ${active}
800
+ `;
801
+
802
+ await using factory = await storageConfig.factory();
803
+ const emittedEvents: { name: string; eventId: string | undefined }[] = [];
804
+ const disposeListener = factory.registerListener({
805
+ replicationEvent: ({ event, event_id }) => emittedEvents.push({ name: event.name, eventId: event_id })
806
+ });
807
+ try {
808
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion }));
809
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
810
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
811
+ const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
812
+ const firstResolved = await firstWriter.resolveTables({
813
+ connection_id: 1,
814
+ source,
815
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38352')
816
+ });
817
+ const oldEventId = [...firstResolved.tables[0].eventDefinitionIds!][0];
818
+ await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1');
819
+ await firstWriter.markAllSnapshotDone('1/1');
820
+ await firstWriter.commit('1/1');
821
+
822
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(false), { storageVersion }));
823
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
824
+ const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage;
825
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
826
+ const resolved = await secondWriter.resolveTables({
827
+ connection_id: 1,
828
+ source,
829
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38353')
830
+ });
831
+
832
+ expect(resolved.tables).toHaveLength(2);
833
+ const oldTable = resolved.tables.find((table) => table.eventDefinitionIds?.has(oldEventId))!;
834
+ const newTable = resolved.tables.find((table) => !table.eventDefinitionIds?.has(oldEventId))!;
835
+ const newEventId = [...newTable.eventDefinitionIds!][0];
836
+ expect(newEventId).toBeDefined();
837
+ expect(newEventId).not.toBe(oldEventId);
838
+ expect(oldTable.snapshotComplete).toBe(true);
839
+ expect(newTable.snapshotComplete).toBe(false);
840
+ expect(newTable.bucketDataSources).toEqual([]);
841
+ expect(newTable.parameterLookupSources).toEqual([]);
842
+
843
+ await expect(secondWriter.markSnapshotDone('2/1')).rejects.toThrow(
844
+ 'Cannot mark snapshot done while source tables still require snapshotting'
845
+ );
846
+
847
+ emittedEvents.length = 0;
848
+ await secondWriter.save({
849
+ sourceTable: newTable,
850
+ tag: storage.SaveOperationTag.INSERT,
851
+ after: { id: 'checkpoint-2', user_id: 'user-2', checkpoint: 2n, active: 0 },
852
+ afterReplicaId: test_utils.rid('checkpoint-2')
853
+ });
854
+ expect(emittedEvents).toEqual([{ name: 'write_checkpoints', eventId: newEventId }]);
855
+
856
+ await secondWriter.markTableSnapshotDone([newTable], '2/1');
857
+ await secondWriter.markSnapshotDone('2/1');
858
+ } finally {
859
+ disposeListener();
860
+ }
861
+ });
862
+
518
863
  test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
519
864
  await using factory = await storageConfig.factory();
520
865
  const syncRules = await factory.updateSyncRules(
@@ -531,7 +876,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
531
876
  )
532
877
  );
533
878
  const bucketStorage = factory.getInstance(syncRules);
534
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
879
+ const syncRulesContent = syncRules.syncConfigContent[0];
880
+ const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
535
881
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
536
882
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
537
883
 
@@ -552,7 +898,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
552
898
  const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
553
899
  const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
554
900
  const buckets = await querier.queryDynamicBucketDescriptions({
555
- async getParameterSets(lookups) {
901
+ async getParameterSets(lookups: ScopedParameterLookup[]) {
556
902
  expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
557
903
  expect(lookups[0].indexId).toEqual('1');
558
904
 
@@ -561,33 +907,729 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
561
907
  return parameter_sets;
562
908
  }
563
909
  });
564
- expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]);
910
+ expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]);
565
911
 
566
912
  const mongoFactory = factory as MongoBucketStorage;
567
913
  const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
568
- const currentDataCollections = await db.listSourceRecordCollectionsV3(syncRules.id);
914
+ const currentDataCollections = await db.listSourceRecordCollections(syncRules.replicationStreamId);
569
915
  const currentData = await currentDataCollections[0]?.findOne({});
570
916
  const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
571
917
  expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
572
918
 
573
919
  const bucketCollections = await mongoFactory.db.db
574
- .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
920
+ .listCollections({ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
575
921
  .toArray();
576
922
  expect(
577
- bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`)
923
+ bucketCollections.some(
924
+ (collection) => collection.name === `bucket_data_${syncRules.replicationStreamId}_${firstBucket?.def}`
925
+ )
578
926
  ).toBe(true);
579
927
 
580
- const syncRule = (await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id })) as ReplicationStreamDocumentV3;
928
+ const syncRule = (await mongoFactory.db.sync_rules.findOne({
929
+ _id: syncRules.replicationStreamId
930
+ })) as ReplicationStreamDocumentV3;
581
931
  const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
582
932
  const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
583
933
  expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
584
934
 
585
935
  const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
586
936
  expect(parameterIndexId).toBeDefined();
587
- const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({});
937
+ const parameterEntry = await db.parameterIndex(syncRules.replicationStreamId, parameterIndexId!).findOne({});
588
938
  expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
589
939
  });
590
940
 
941
+ test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => {
942
+ await using factory = await storageConfig.factory();
943
+
944
+ const first = await factory.updateSyncRules(
945
+ updateSyncRulesFromYaml(
946
+ `
947
+ config:
948
+ edition: 2
949
+
950
+ streams:
951
+ by_owner:
952
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
953
+ `,
954
+ { storageVersion }
955
+ )
956
+ );
957
+ const second = await factory.updateSyncRules(
958
+ updateSyncRulesFromYaml(
959
+ `
960
+ config:
961
+ edition: 2
962
+
963
+ streams:
964
+ by_project:
965
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
966
+ `,
967
+ { storageVersion }
968
+ )
969
+ );
970
+
971
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
972
+ expect(
973
+ (await factory.getStoppedReplicationStreams()).find(
974
+ (stream) => stream.replicationStreamId == first.replicationStreamId
975
+ )?.state
976
+ ).toBe(storage.SyncRuleState.STOP);
977
+
978
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
979
+ expect(replicatingStreams).toHaveLength(1);
980
+ expect(replicatingStreams[0].replicationStreamId).toEqual(second.replicationStreamId);
981
+
982
+ const configs = second.syncConfigContent;
983
+ expect(configs).toHaveLength(1);
984
+ const statuses = await Promise.all(configs.map((config) => config.getSyncConfigStatus()));
985
+ expect(statuses.map((status) => status?.state)).toEqual([storage.SyncRuleState.PROCESSING]);
986
+ expect(statuses.map((status) => status?.id)).toEqual(configs.map((config) => config.syncConfigId));
987
+ const parsed = replicatingStreams[0].parsed(test_utils.PARSE_OPTIONS);
988
+ expect(parsed.syncConfigs).toHaveLength(1);
989
+ expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
990
+
991
+ const bucketStorage = factory.getInstance(replicatingStreams[0]);
992
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
993
+ const resolved = await writer.resolveTables({
994
+ connection_id: 1,
995
+ source: sourceDescriptor('todos', { objectId: 'todos-relation' }),
996
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834b')
997
+ });
998
+ expect(resolved.tables).toHaveLength(1);
999
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
1000
+ });
1001
+
1002
+ test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => {
1003
+ await using factory = await storageConfig.factory();
1004
+
1005
+ const first = await factory.updateSyncRules(
1006
+ updateSyncRulesFromYaml(
1007
+ `
1008
+ config:
1009
+ edition: 3
1010
+
1011
+ streams:
1012
+ by_owner:
1013
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1014
+ `,
1015
+ { storageVersion }
1016
+ )
1017
+ );
1018
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1019
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1020
+ await firstWriter.markAllSnapshotDone('1/1');
1021
+ await firstWriter.commit('1/1');
1022
+
1023
+ const second = await factory.updateSyncRules(
1024
+ updateSyncRulesFromYaml(
1025
+ `
1026
+ config:
1027
+ edition: 3
1028
+
1029
+ streams:
1030
+ by_project:
1031
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1032
+ `,
1033
+ { storageVersion }
1034
+ )
1035
+ );
1036
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1037
+
1038
+ const bucketStorage = factory.getInstance(second);
1039
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1040
+ const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
1041
+
1042
+ const resolved = await writer.resolveTables({
1043
+ connection_id: 1,
1044
+ source,
1045
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834c')
1046
+ });
1047
+ expect(resolved.tables).toHaveLength(1);
1048
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(2);
1049
+
1050
+ await writer.markAllSnapshotDone('2/1');
1051
+ await writer.commit('2/1');
1052
+
1053
+ const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1054
+ await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
1055
+ const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
1056
+ expect(activeStatus?.bucketDataSources).toHaveLength(1);
1057
+ });
1058
+
1059
+ test.runIf(storageVersion >= 3)(
1060
+ 'editing a stream query on the same table does not drop the active source table',
1061
+ async () => {
1062
+ await using factory = await storageConfig.factory();
1063
+
1064
+ const first = await factory.updateSyncRules(
1065
+ updateSyncRulesFromYaml(
1066
+ `
1067
+ config:
1068
+ edition: 3
1069
+
1070
+ streams:
1071
+ todos:
1072
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1073
+ `,
1074
+ { storageVersion }
1075
+ )
1076
+ );
1077
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1078
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1079
+ const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
1080
+ const firstResolved = await firstWriter.resolveTables({
1081
+ connection_id: 1,
1082
+ source,
1083
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
1084
+ });
1085
+ expect(firstResolved.dropTables).toHaveLength(0);
1086
+ await firstWriter.markAllSnapshotDone('1/1');
1087
+ await firstWriter.commit('1/1');
1088
+
1089
+ const second = await factory.updateSyncRules(
1090
+ updateSyncRulesFromYaml(
1091
+ `
1092
+ config:
1093
+ edition: 3
1094
+
1095
+ streams:
1096
+ todos:
1097
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1098
+ `,
1099
+ { storageVersion }
1100
+ )
1101
+ );
1102
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1103
+
1104
+ const deploying = await factory.getDeployingSyncConfig();
1105
+ expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
1106
+ const bucketStorage = deploying!.storage;
1107
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1108
+ const resolved = await writer.resolveTables({
1109
+ connection_id: 1,
1110
+ source,
1111
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834e')
1112
+ });
1113
+
1114
+ expect(resolved.dropTables).toHaveLength(0);
1115
+ expect(resolved.tables.map((table) => table.id.toString())).toEqual(['6544e3899293153fa7b3834e']);
1116
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
1117
+
1118
+ const activeStatus = await firstWriter.getSourceTableStatus(firstResolved.tables[0]);
1119
+ expect(activeStatus?.bucketDataSources).toHaveLength(1);
1120
+ }
1121
+ );
1122
+
1123
+ test.runIf(storageVersion >= 3)('reserves historical mapping ids without reusing stopped configs', async () => {
1124
+ await using factory = await storageConfig.factory();
1125
+
1126
+ const ownerRules = `
1127
+ config:
1128
+ edition: 3
1129
+
1130
+ streams:
1131
+ by_owner:
1132
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1133
+ `;
1134
+ const projectRules = `
1135
+ config:
1136
+ edition: 3
1137
+
1138
+ streams:
1139
+ by_project:
1140
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1141
+ `;
1142
+ const statusRules = `
1143
+ config:
1144
+ edition: 3
1145
+
1146
+ streams:
1147
+ by_status:
1148
+ query: SELECT * FROM todos WHERE status = subscription.parameter('status')
1149
+ `;
1150
+
1151
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
1152
+ const firstStorage = factory.getInstance(first);
1153
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1154
+ await firstWriter.markAllSnapshotDone('1/1');
1155
+ await firstWriter.commit('1/1');
1156
+
1157
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(projectRules, { storageVersion }));
1158
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1159
+
1160
+ const third = await factory.updateSyncRules(updateSyncRulesFromYaml(statusRules, { storageVersion }));
1161
+ expect(third.replicationStreamId).toBe(first.replicationStreamId);
1162
+
1163
+ const configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1164
+ const statuses = await Promise.all(
1165
+ configs.map(async (config) => [config, await config.getSyncConfigStatus()] as const)
1166
+ );
1167
+ // ownerRules
1168
+ const activeConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.ACTIVE)?.[0];
1169
+ // projectRules
1170
+ const stoppedConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.STOP)?.[0];
1171
+ // statusRules
1172
+ const processingConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.PROCESSING)?.[0];
1173
+ expect(stoppedConfig).toBeDefined();
1174
+ expect(activeConfig).toBeDefined();
1175
+ expect(processingConfig).toBeDefined();
1176
+
1177
+ const activePrefix = bucketRequest(activeConfig!, 'by_owner|0["owner"]').bucket;
1178
+ const stoppedPrefix = bucketRequest(stoppedConfig!, 'by_project|0["project"]').bucket;
1179
+ const processingPrefix = bucketRequest(processingConfig!, 'by_status|0["status"]').bucket;
1180
+
1181
+ expect(processingPrefix).not.toBe(stoppedPrefix);
1182
+ expect(processingPrefix).not.toBe(activePrefix);
1183
+ expect(bucketDefinitionId(processingPrefix)).toBeGreaterThan(
1184
+ Math.max(bucketDefinitionId(stoppedPrefix), bucketDefinitionId(activePrefix))
1185
+ );
1186
+ });
1187
+
1188
+ test.runIf(storageVersion >= 3)('table snapshot status only affects sync configs using that table', async () => {
1189
+ await using factory = await storageConfig.factory();
1190
+
1191
+ const first = await factory.updateSyncRules(
1192
+ updateSyncRulesFromYaml(
1193
+ `
1194
+ config:
1195
+ edition: 3
1196
+
1197
+ streams:
1198
+ by_owner:
1199
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1200
+ `,
1201
+ { storageVersion }
1202
+ )
1203
+ );
1204
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1205
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1206
+ await firstWriter.markAllSnapshotDone('1/1');
1207
+ await firstWriter.commit('1/1');
1208
+
1209
+ const second = await factory.updateSyncRules(
1210
+ updateSyncRulesFromYaml(
1211
+ `
1212
+ config:
1213
+ edition: 3
1214
+
1215
+ streams:
1216
+ by_owner:
1217
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1218
+ by_project:
1219
+ query: SELECT * FROM projects WHERE id = subscription.parameter('project_id')
1220
+ `,
1221
+ { storageVersion }
1222
+ )
1223
+ );
1224
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1225
+
1226
+ const bucketStorage = factory.getInstance(second);
1227
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1228
+ const resolved = await writer.resolveTables({
1229
+ connection_id: 1,
1230
+ source: sourceDescriptor('projects', { objectId: 'projects-relation' }),
1231
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
1232
+ });
1233
+ expect(resolved.tables).toHaveLength(1);
1234
+
1235
+ await writer.markAllSnapshotDone('2/1');
1236
+ await writer.markTableSnapshotRequired(resolved.tables[0]);
1237
+
1238
+ const configsAfterRequired = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1239
+ const statusesAfterRequired = await Promise.all(configsAfterRequired.map((config) => config.getSyncConfigStatus()));
1240
+ const ownerConfigAfterRequired = statusesAfterRequired.find(
1241
+ (status) => status?.state == storage.SyncRuleState.ACTIVE
1242
+ );
1243
+ const projectConfigAfterRequired = statusesAfterRequired.find(
1244
+ (status) => status?.state == storage.SyncRuleState.PROCESSING
1245
+ );
1246
+ expect(ownerConfigAfterRequired?.snapshot_done).toBe(true);
1247
+ expect(projectConfigAfterRequired?.snapshot_done).toBe(false);
1248
+
1249
+ await writer.markTableSnapshotDone(resolved.tables, '3/1');
1250
+
1251
+ const configsAfterDone = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1252
+ const statusesAfterDone = await Promise.all(configsAfterDone.map((config) => config.getSyncConfigStatus()));
1253
+ const ownerConfigAfterDone = statusesAfterDone.find((status) => status?.id == ownerConfigAfterRequired!.id);
1254
+ const projectConfigAfterDone = statusesAfterDone.find((status) => status?.id == projectConfigAfterRequired!.id);
1255
+ expect(ownerConfigAfterDone?.snapshot_done).toBe(true);
1256
+ expect(projectConfigAfterDone?.snapshot_done).toBe(false);
1257
+ });
1258
+
1259
+ test.runIf(storageVersion >= 3)(
1260
+ 'keeps compatible active and deploying sync configs in one replication stream',
1261
+ async () => {
1262
+ await using factory = await storageConfig.factory();
1263
+
1264
+ const first = await factory.updateSyncRules(
1265
+ updateSyncRulesFromYaml(
1266
+ `
1267
+ config:
1268
+ edition: 3
1269
+
1270
+ streams:
1271
+ by_owner:
1272
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1273
+ `,
1274
+ { storageVersion }
1275
+ )
1276
+ );
1277
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1278
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1279
+ await firstWriter.markAllSnapshotDone('1/1');
1280
+ await firstWriter.commit('1/1');
1281
+ const initialActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1282
+
1283
+ let configs = first.syncConfigContent;
1284
+ const firstConfigId = configs[0].syncConfigId;
1285
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
1286
+
1287
+ const second = await factory.updateSyncRules(
1288
+ updateSyncRulesFromYaml(
1289
+ `
1290
+ config:
1291
+ edition: 3
1292
+
1293
+ streams:
1294
+ by_project:
1295
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1296
+ `,
1297
+ { storageVersion }
1298
+ )
1299
+ );
1300
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1301
+
1302
+ configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1303
+ const deploying = await factory.getDeployingSyncConfig();
1304
+ expect(configs).toHaveLength(2);
1305
+ expect(deploying).not.toBeNull();
1306
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
1307
+
1308
+ const stream = (await factory.getActiveSyncConfig())?.replicationStream;
1309
+ expect(stream?.state).toBe(storage.SyncRuleState.ACTIVE);
1310
+ expect(
1311
+ (await Promise.all(configs.map((config) => config.getSyncConfigStatus()))).map((status) => status?.state).sort()
1312
+ ).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING]);
1313
+
1314
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1315
+ expect(replicatingStreams).toHaveLength(1);
1316
+ for (const config of configs) {
1317
+ expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
1318
+ }
1319
+
1320
+ const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
1321
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
1322
+ await secondWriter.markAllSnapshotDone('2/1');
1323
+ await secondWriter.commit('2/1');
1324
+
1325
+ const updatedStream = (await factory.getActiveSyncConfig())?.replicationStream;
1326
+ expect(updatedStream?.state).toBe(storage.SyncRuleState.ACTIVE);
1327
+ const updatedConfigs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1328
+ expect(
1329
+ (await Promise.all(updatedConfigs.map((config) => config.getSyncConfigStatus())))
1330
+ .map((status) => status?.state)
1331
+ .sort()
1332
+ ).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.STOP]);
1333
+ expect(await factory.getDeployingSyncConfig()).toBeNull();
1334
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).not.toBe(firstConfigId);
1335
+ const updatedActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1336
+ expect(updatedActiveStorage.replicationStream.replicationJobId).not.toBe(
1337
+ initialActiveStorage.replicationStream.replicationJobId
1338
+ );
1339
+ expect(updatedActiveStorage.replicationStream.replicationJobId).toContain(
1340
+ (await factory.getActiveSyncConfig())!.content.syncConfigId
1341
+ );
1342
+ }
1343
+ );
1344
+
1345
+ test.runIf(storageVersion >= 3)(
1346
+ 'restart creates a replacement stream instead of appending to the active stream',
1347
+ async () => {
1348
+ await using factory = await storageConfig.factory();
1349
+
1350
+ const active = await factory.updateSyncRules(
1351
+ updateSyncRulesFromYaml(
1352
+ `
1353
+ config:
1354
+ edition: 3
1355
+
1356
+ streams:
1357
+ by_owner:
1358
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1359
+ `,
1360
+ { storageVersion }
1361
+ )
1362
+ );
1363
+ const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage;
1364
+ await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
1365
+ await activeWriter.markAllSnapshotDone('1/1');
1366
+ await activeWriter.commit('1/1');
1367
+
1368
+ await factory.restartReplication(active.replicationStreamId);
1369
+
1370
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1371
+ expect(replicatingStreams).toHaveLength(1);
1372
+ expect(replicatingStreams[0].replicationStreamId).not.toEqual(active.replicationStreamId);
1373
+ expect(replicatingStreams[0].state).toBe(storage.SyncRuleState.PROCESSING);
1374
+
1375
+ const oldStream = (await (factory as MongoBucketStorage).db.sync_rules.findOne({
1376
+ _id: active.replicationStreamId
1377
+ })) as ReplicationStreamDocumentV3;
1378
+ expect(oldStream.state).toBe(storage.SyncRuleState.ERRORED);
1379
+ expect(oldStream.sync_configs.map((config) => config.state)).toEqual([storage.SyncRuleState.ERRORED]);
1380
+ }
1381
+ );
1382
+
1383
+ test.runIf(storageVersion >= 3)('restart errors an active stream with an embedded deploying config', async () => {
1384
+ await using factory = await storageConfig.factory();
1385
+
1386
+ const active = await factory.updateSyncRules(
1387
+ updateSyncRulesFromYaml(
1388
+ `
1389
+ config:
1390
+ edition: 3
1391
+
1392
+ streams:
1393
+ by_owner:
1394
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1395
+ `,
1396
+ { storageVersion }
1397
+ )
1398
+ );
1399
+ const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage;
1400
+ await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
1401
+ await activeWriter.markAllSnapshotDone('1/1');
1402
+ await activeWriter.commit('1/1');
1403
+
1404
+ const deploying = await factory.updateSyncRules(
1405
+ updateSyncRulesFromYaml(
1406
+ `
1407
+ config:
1408
+ edition: 3
1409
+
1410
+ streams:
1411
+ by_project:
1412
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1413
+ `,
1414
+ { storageVersion }
1415
+ )
1416
+ );
1417
+ expect(deploying.replicationStreamId).toEqual(active.replicationStreamId);
1418
+
1419
+ await factory.restartReplication(active.replicationStreamId);
1420
+
1421
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1422
+ expect(replicatingStreams).toHaveLength(1);
1423
+ expect(replicatingStreams[0].replicationStreamId).not.toEqual(active.replicationStreamId);
1424
+
1425
+ const oldStream = (await (factory as MongoBucketStorage).db.sync_rules.findOne({
1426
+ _id: active.replicationStreamId
1427
+ })) as ReplicationStreamDocumentV3;
1428
+ expect(oldStream.state).toBe(storage.SyncRuleState.ERRORED);
1429
+ expect(oldStream.sync_configs.map((config) => config.state)).toEqual([
1430
+ storage.SyncRuleState.ERRORED,
1431
+ storage.SyncRuleState.STOP
1432
+ ]);
1433
+ const oldActive = await factory.getActiveSyncConfig();
1434
+ expect(oldActive).not.toBeNull();
1435
+ expect(oldActive!.replicationStream.syncConfigContent).toHaveLength(1);
1436
+ await expect(oldActive!.storage.getCheckpoint()).resolves.toBeDefined();
1437
+ });
1438
+
1439
+ test.runIf(storageVersion >= 3)(
1440
+ 'appended compatible config adopts the stream checkpoint instead of regressing to 0',
1441
+ async () => {
1442
+ await using factory = await storageConfig.factory();
1443
+
1444
+ // First config replicates some data, advancing the stream-level op head and its checkpoint.
1445
+ const first = await factory.updateSyncRules(
1446
+ updateSyncRulesFromYaml(
1447
+ `
1448
+ config:
1449
+ edition: 3
1450
+
1451
+ streams:
1452
+ by_owner:
1453
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1454
+ `,
1455
+ { storageVersion }
1456
+ )
1457
+ );
1458
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1459
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1460
+ const sourceTable = await test_utils.resolveTestTable(
1461
+ firstWriter,
1462
+ 'todos',
1463
+ ['id'],
1464
+ INITIALIZED_MONGO_STORAGE_FACTORY
1465
+ );
1466
+ await firstWriter.save({
1467
+ sourceTable,
1468
+ tag: storage.SaveOperationTag.INSERT,
1469
+ after: {
1470
+ id: 'todo-1',
1471
+ owner_id: 'user-1',
1472
+ project_id: 'project-1'
1473
+ },
1474
+ afterReplicaId: test_utils.rid('todo-1')
1475
+ });
1476
+ await firstWriter.markAllSnapshotDone('1/1');
1477
+ await firstWriter.commit('1/1');
1478
+
1479
+ const firstCheckpoint = (await firstStorage.getCheckpoint()).checkpoint;
1480
+ expect(firstCheckpoint).toBeGreaterThan(0n);
1481
+
1482
+ const mongoFactory = factory as MongoBucketStorage;
1483
+ const streamDocBefore = (await mongoFactory.db.sync_rules.findOne({
1484
+ _id: first.replicationStreamId
1485
+ })) as ReplicationStreamDocumentV3;
1486
+ // The stream-level head was advanced durably to (at least) the first config's checkpoint.
1487
+ expect(streamDocBefore.last_persisted_op).not.toBeNull();
1488
+ expect(BigInt(streamDocBefore.last_persisted_op!)).toBeGreaterThanOrEqual(firstCheckpoint);
1489
+
1490
+ // Append a compatible second config that replicates nothing new.
1491
+ const second = await factory.updateSyncRules(
1492
+ updateSyncRulesFromYaml(
1493
+ `
1494
+ config:
1495
+ edition: 3
1496
+
1497
+ streams:
1498
+ by_project:
1499
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1500
+ `,
1501
+ { storageVersion }
1502
+ )
1503
+ );
1504
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1505
+
1506
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1507
+ expect(replicatingStreams).toHaveLength(1);
1508
+ const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
1509
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
1510
+ // No new data replicated - just complete the snapshot and commit.
1511
+ await secondWriter.markAllSnapshotDone('2/1');
1512
+ await secondWriter.commit('2/1');
1513
+
1514
+ // The appended config's checkpoint must adopt the stream head, not regress to 0.
1515
+ const streamDocAfter = (await mongoFactory.db.sync_rules.findOne({
1516
+ _id: first.replicationStreamId
1517
+ })) as ReplicationStreamDocumentV3;
1518
+ const head = BigInt(streamDocAfter.last_persisted_op!);
1519
+ for (const config of streamDocAfter.sync_configs) {
1520
+ expect(config.last_checkpoint).not.toBeNull();
1521
+ expect(BigInt(config.last_checkpoint!)).toEqual(head);
1522
+ }
1523
+ // All configs share the same checkpoint, equal to the stream head.
1524
+ expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
1525
+
1526
+ // After activation, the active config's checkpoint does not regress.
1527
+ const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1528
+ const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
1529
+ expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
1530
+ }
1531
+ );
1532
+
1533
+ test.runIf(storageVersion >= 3)('creates a new replication stream when compatibility options differ', async () => {
1534
+ await using factory = await storageConfig.factory();
1535
+
1536
+ const first = await factory.updateSyncRules(
1537
+ updateSyncRulesFromYaml(
1538
+ `
1539
+ config:
1540
+ edition: 3
1541
+
1542
+ streams:
1543
+ by_owner:
1544
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1545
+ `,
1546
+ { storageVersion }
1547
+ )
1548
+ );
1549
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1550
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1551
+ await firstWriter.markAllSnapshotDone('1/1');
1552
+ await firstWriter.commit('1/1');
1553
+
1554
+ // Same streams, but a different compatibility edition - must not append to the active stream.
1555
+ const second = await factory.updateSyncRules(
1556
+ updateSyncRulesFromYaml(
1557
+ `
1558
+ config:
1559
+ edition: 2
1560
+
1561
+ streams:
1562
+ by_owner:
1563
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1564
+ `,
1565
+ { storageVersion }
1566
+ )
1567
+ );
1568
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1569
+
1570
+ // The first stream stays active until the replacement has replicated.
1571
+ const active = await factory.getActiveSyncConfig();
1572
+ expect(active?.replicationStream.replicationStreamId).toEqual(first.replicationStreamId);
1573
+ expect((await factory.getDeployingSyncConfig())?.replicationStream.replicationStreamId).toEqual(
1574
+ second.replicationStreamId
1575
+ );
1576
+ });
1577
+
1578
+ test.runIf(storageVersion >= 3)(
1579
+ 'creates a new replication stream for legacy sync rules without a serialized plan',
1580
+ async () => {
1581
+ await using factory = await storageConfig.factory();
1582
+
1583
+ const first = await factory.updateSyncRules(
1584
+ updateSyncRulesFromYaml(
1585
+ `
1586
+ config:
1587
+ edition: 3
1588
+
1589
+ streams:
1590
+ by_owner:
1591
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1592
+ `,
1593
+ { storageVersion }
1594
+ )
1595
+ );
1596
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1597
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1598
+ await firstWriter.markAllSnapshotDone('1/1');
1599
+ await firstWriter.commit('1/1');
1600
+
1601
+ // Legacy sync rules have no serialized plan - must not append to the active stream.
1602
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
1603
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1604
+ }
1605
+ );
1606
+
1607
+ test.runIf(storageVersion >= 3)('does not append to an active legacy sync config', async () => {
1608
+ await using factory = await storageConfig.factory();
1609
+
1610
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
1611
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1612
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1613
+ await firstWriter.markAllSnapshotDone('1/1');
1614
+ await firstWriter.commit('1/1');
1615
+
1616
+ // The active config has no serialized plan, so a new streams config must not be appended to it.
1617
+ const second = await factory.updateSyncRules(
1618
+ updateSyncRulesFromYaml(
1619
+ `
1620
+ config:
1621
+ edition: 3
1622
+
1623
+ streams:
1624
+ by_owner:
1625
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1626
+ `,
1627
+ { storageVersion }
1628
+ )
1629
+ );
1630
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1631
+ });
1632
+
591
1633
  test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
592
1634
  await using factory = await storageConfig.factory();
593
1635
 
@@ -600,7 +1642,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
600
1642
  ).resolves.toBeDefined();
601
1643
 
602
1644
  const mongoFactory = factory as MongoBucketStorage;
603
- expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.id }))?.state).toBe(
1645
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.replicationStreamId }))?.state).toBe(
604
1646
  storage.SyncRuleState.STOP
605
1647
  );
606
1648
  });
@@ -612,10 +1654,48 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
612
1654
  updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
613
1655
  );
614
1656
 
615
- expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.id);
1657
+ expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.replicationStreamId);
616
1658
  await syncRules.current_lock?.release();
617
1659
  });
618
1660
 
1661
+ test.runIf(storageVersion >= 3)('does not lock when appending a sync config to an existing stream', async () => {
1662
+ await using factory = await storageConfig.factory();
1663
+
1664
+ const firstRules = `
1665
+ config:
1666
+ edition: 3
1667
+
1668
+ streams:
1669
+ by_owner:
1670
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1671
+ `;
1672
+ const secondRules = `
1673
+ config:
1674
+ edition: 3
1675
+
1676
+ streams:
1677
+ by_project:
1678
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1679
+ `;
1680
+
1681
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstRules, { storageVersion, lock: true }));
1682
+ expect(first.current_lock?.sync_rules_id).toBe(first.replicationStreamId);
1683
+ try {
1684
+ const firstStorage = factory.getInstance(first);
1685
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1686
+ await firstWriter.markAllSnapshotDone('1/1');
1687
+ await firstWriter.commit('1/1');
1688
+
1689
+ const second = await factory.updateSyncRules(
1690
+ updateSyncRulesFromYaml(secondRules, { storageVersion, lock: true })
1691
+ );
1692
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1693
+ expect(second.current_lock).toBeNull();
1694
+ } finally {
1695
+ await first.current_lock?.release();
1696
+ }
1697
+ });
1698
+
619
1699
  test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
620
1700
  await using factory = await storageConfig.factory();
621
1701
  const syncRules = await factory.updateSyncRules(
@@ -647,10 +1727,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
647
1727
  await writer.commit('1/1');
648
1728
 
649
1729
  const mongoFactory = factory as MongoBucketStorage;
650
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1730
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
651
1731
 
652
1732
  const sourceRecordCollections = await mongoFactory.db.db
653
- .listCollections({ name: new RegExp(`^source_records_${syncRules.id}_`) }, { nameOnly: true })
1733
+ .listCollections({ name: new RegExp(`^source_records_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
654
1734
  .toArray();
655
1735
  expect(sourceRecordCollections).toEqual([]);
656
1736
  });
@@ -686,13 +1766,103 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
686
1766
  await writer.commit('1/1');
687
1767
 
688
1768
  const mongoFactory = factory as MongoBucketStorage;
689
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1769
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
690
1770
 
691
1771
  await bucketStorage.clear();
692
1772
 
693
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0);
1773
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
694
1774
  });
695
1775
 
1776
+ test.runIf(storageVersion < 3)(
1777
+ 'clear removes large v1 current_data sets without affecting other streams',
1778
+ async () => {
1779
+ await using factory = await storageConfig.factory();
1780
+ const syncRules = await factory.updateSyncRules(
1781
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
1782
+ storageVersion
1783
+ })
1784
+ );
1785
+ const bucketStorage = factory.getInstance(syncRules);
1786
+ const mongoFactory = factory as MongoBucketStorage;
1787
+ const sourceTableId = new bson.ObjectId();
1788
+ const documents = Array.from({ length: 10_002 }, (_, index) => ({
1789
+ _id: {
1790
+ g: syncRules.replicationStreamId,
1791
+ t: sourceTableId,
1792
+ k: index
1793
+ },
1794
+ data: new bson.Binary(),
1795
+ buckets: [],
1796
+ lookups: []
1797
+ }));
1798
+ await mongoFactory.db.current_data.insertMany([
1799
+ ...documents,
1800
+ {
1801
+ _id: {
1802
+ g: syncRules.replicationStreamId + 1,
1803
+ t: sourceTableId,
1804
+ k: 0
1805
+ },
1806
+ data: new bson.Binary(),
1807
+ buckets: [],
1808
+ lookups: []
1809
+ }
1810
+ ]);
1811
+
1812
+ await bucketStorage.clear();
1813
+
1814
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
1815
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId + 1 })).toBe(1);
1816
+ }
1817
+ );
1818
+
1819
+ test.runIf(storageVersion < 3)(
1820
+ 'clear removes large v1 parameter index sets without affecting other streams',
1821
+ async () => {
1822
+ await using factory = await storageConfig.factory();
1823
+ const syncRules = await factory.updateSyncRules(
1824
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
1825
+ storageVersion
1826
+ })
1827
+ );
1828
+ const bucketStorage = factory.getInstance(syncRules);
1829
+ const mongoFactory = factory as MongoBucketStorage;
1830
+ const sourceTableId = new bson.ObjectId();
1831
+ const documents = Array.from({ length: 10_002 }, (_, index) => ({
1832
+ _id: BigInt(index * 2),
1833
+ key: {
1834
+ g: syncRules.replicationStreamId,
1835
+ t: sourceTableId,
1836
+ k: index
1837
+ },
1838
+ lookup: new bson.Binary(),
1839
+ bucket_parameters: []
1840
+ }));
1841
+ await mongoFactory.db.bucket_parameters.insertMany([
1842
+ ...documents,
1843
+ {
1844
+ _id: 1n,
1845
+ key: {
1846
+ g: syncRules.replicationStreamId + 1,
1847
+ t: sourceTableId,
1848
+ k: 0
1849
+ },
1850
+ lookup: new bson.Binary(),
1851
+ bucket_parameters: []
1852
+ }
1853
+ ]);
1854
+
1855
+ await bucketStorage.clear();
1856
+
1857
+ expect(await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId })).toBe(
1858
+ 0
1859
+ );
1860
+ expect(
1861
+ await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId + 1 })
1862
+ ).toBe(1);
1863
+ }
1864
+ );
1865
+
696
1866
  test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
697
1867
  await using factory = await storageConfig.factory();
698
1868
  const syncRules = await factory.updateSyncRules(
@@ -725,12 +1895,28 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
725
1895
  await writer.commit('1/1');
726
1896
 
727
1897
  const mongoFactory = factory as MongoBucketStorage;
728
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1898
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
729
1899
 
730
1900
  const metricsAfter = await factory.getStorageMetrics();
731
1901
  expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
732
1902
  });
733
1903
 
1904
+ test.runIf(storageVersion >= 3)('storage metrics include the sync config version label', async () => {
1905
+ await using factory = await storageConfig.factory();
1906
+ const syncRules = await factory.updateSyncRules(
1907
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, version_label: 'metrics-test' })
1908
+ );
1909
+
1910
+ const metrics = await factory.getStorageMetrics();
1911
+
1912
+ expect(metrics.sync_config_metrics).toContainEqual(
1913
+ expect.objectContaining({
1914
+ sync_config_id: syncRules.syncConfigContent[0].syncConfigId,
1915
+ version_label: 'metrics-test'
1916
+ })
1917
+ );
1918
+ });
1919
+
734
1920
  test.runIf(storageVersion >= 3)(
735
1921
  'loads parameter checkpoint changes across all v3 parameter index collections',
736
1922
  async () => {
@@ -798,25 +1984,25 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
798
1984
  );
799
1985
 
800
1986
  const mongoFactory = factory as MongoBucketStorage;
801
- const bucketStorage = mongoFactory.getInstance(syncRules) as any;
802
- const db = bucketStorage.db;
803
- await db.initializeStreamStorage(syncRules.id);
1987
+ const bucketStorage = mongoFactory.getInstance(syncRules);
1988
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1989
+ await db.initializeStreamStorage(syncRules.replicationStreamId);
804
1990
 
805
1991
  const sourceTableA = new bson.ObjectId();
806
1992
  const sourceTableB = new bson.ObjectId();
807
- await db.sourceTablesV3(syncRules.id).insertMany([
1993
+ await db.sourceTables(syncRules.replicationStreamId).insertMany([
808
1994
  {
809
1995
  _id: sourceTableA,
810
1996
  connection_id: 1,
811
1997
  relation_id: 'a',
812
1998
  schema_name: 'public',
813
1999
  table_name: 'table_a',
814
- replica_id_columns: null,
815
- replica_id_columns2: [],
2000
+ replica_id_columns: [],
816
2001
  snapshot_done: true,
817
2002
  snapshot_status: undefined,
818
2003
  bucket_data_source_ids: [],
819
2004
  parameter_lookup_source_ids: [],
2005
+ event_definition_ids: [],
820
2006
  latest_pending_delete: 9n
821
2007
  },
822
2008
  {
@@ -825,45 +2011,63 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
825
2011
  relation_id: 'b',
826
2012
  schema_name: 'public',
827
2013
  table_name: 'table_b',
828
- replica_id_columns: null,
829
- replica_id_columns2: [],
2014
+ replica_id_columns: [],
830
2015
  snapshot_done: true,
831
2016
  snapshot_status: undefined,
832
2017
  bucket_data_source_ids: [],
833
2018
  parameter_lookup_source_ids: [],
2019
+ event_definition_ids: [],
834
2020
  latest_pending_delete: 12n
835
2021
  }
836
2022
  ]);
837
2023
 
838
- await db.sourceRecordsV3(syncRules.id, sourceTableA).insertMany([
2024
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).insertMany([
839
2025
  { _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
840
2026
  { _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
841
2027
  { _id: 'active', data: null, buckets: [], lookups: [] }
842
2028
  ]);
843
2029
  await db
844
- .sourceRecordsV3(syncRules.id, sourceTableB)
2030
+ .sourceRecords(syncRules.replicationStreamId, sourceTableB)
845
2031
  .insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
846
2032
 
847
- const store = new SourceRecordStoreV3(db, syncRules.id, bucketStorage.sync_rules.mapping);
2033
+ const store = new SourceRecordStoreV3(
2034
+ db,
2035
+ syncRules.replicationStreamId,
2036
+ bucketStorage.replicationStream.syncConfigContent[0].mapping
2037
+ );
848
2038
  const logger = { info() {} } as any;
849
2039
 
850
2040
  await store.postCommitCleanup(6n, logger);
851
2041
 
852
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 5n })).toBe(0);
853
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 9n })).toBe(1);
854
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableB).countDocuments({ pending_delete: 12n })).toBe(1);
855
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete).toBe(9n);
856
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
2042
+ expect(
2043
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 5n })
2044
+ ).toBe(0);
2045
+ expect(
2046
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 9n })
2047
+ ).toBe(1);
2048
+ expect(
2049
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableB).countDocuments({ pending_delete: 12n })
2050
+ ).toBe(1);
2051
+ expect(
2052
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
2053
+ ).toBe(9n);
2054
+ expect(
2055
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
2056
+ ).toBe(12n);
857
2057
 
858
2058
  await store.postCommitCleanup(10n, logger);
859
2059
 
860
2060
  expect(
861
- await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: { $exists: true } })
2061
+ await db
2062
+ .sourceRecords(syncRules.replicationStreamId, sourceTableA)
2063
+ .countDocuments({ pending_delete: { $exists: true } })
862
2064
  ).toBe(0);
863
2065
  expect(
864
- (await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete
2066
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
865
2067
  ).toBeUndefined();
866
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
2068
+ expect(
2069
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
2070
+ ).toBe(12n);
867
2071
  });
868
2072
  }
869
2073
 
@@ -880,7 +2084,7 @@ describe('sync - mongodb', () => {
880
2084
  await legacyWriter.markAllSnapshotDone('1/1');
881
2085
  await legacyWriter.commit('1/1');
882
2086
 
883
- expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
2087
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
884
2088
  storage.SyncRuleState.ACTIVE
885
2089
  );
886
2090
 
@@ -892,7 +2096,7 @@ describe('sync - mongodb', () => {
892
2096
  await v3Writer.markAllSnapshotDone('2/1');
893
2097
  await v3Writer.commit('2/1');
894
2098
 
895
- expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
2099
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
896
2100
  storage.SyncRuleState.STOP
897
2101
  );
898
2102
  });
@@ -900,6 +2104,233 @@ describe('sync - mongodb', () => {
900
2104
  for (const storageVersion of TEST_STORAGE_VERSIONS) {
901
2105
  describe(`storage v${storageVersion}`, () => {
902
2106
  registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
2107
+
2108
+ describe.runIf(storageVersion >= 3)('V3 read filtering boundaries', () => {
2109
+ async function setupFilteringTest() {
2110
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
2111
+ const syncRules = await factory.updateSyncRules(
2112
+ updateSyncRulesFromYaml(
2113
+ `
2114
+ bucket_definitions:
2115
+ global:
2116
+ data:
2117
+ - SELECT id, description FROM test
2118
+ `,
2119
+ { storageVersion }
2120
+ )
2121
+ );
2122
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
2123
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
2124
+ const mapping = syncRules.syncConfigContent[0].mapping;
2125
+
2126
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
2127
+ const definitionId = mapping.bucketSourceId(request.source);
2128
+ const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
2129
+
2130
+ const bucketName = request.bucket;
2131
+ const sourceTable = new bson.ObjectId();
2132
+ const bucketKey: BucketKey = {
2133
+ replicationStreamId: syncRules.replicationStreamId,
2134
+ definitionId,
2135
+ bucket: bucketName
2136
+ };
2137
+
2138
+ function makeOps(opIds: bigint[]): BucketDataDoc[] {
2139
+ return opIds.map((opId) => ({
2140
+ bucketKey,
2141
+ o: opId,
2142
+ op: 'PUT' as const,
2143
+ source_table: sourceTable,
2144
+ source_key: test_utils.rid(`row-${opId}`),
2145
+ table: 'items',
2146
+ row_id: `row-${opId}`,
2147
+ checksum: BigInt(opId) * 10n,
2148
+ data: `{"id":"row-${opId}"}`
2149
+ }));
2150
+ }
2151
+
2152
+ const docA = serializeBucketData(bucketName, makeOps([10n, 20n, 30n]));
2153
+ const docB = serializeBucketData(bucketName, makeOps([40n, 50n, 60n]));
2154
+ const docC = serializeBucketData(bucketName, makeOps([70n, 80n, 90n]));
2155
+
2156
+ await collection.insertMany([docA, docB, docC]);
2157
+
2158
+ return { factory, syncRules, bucketStorage, bucketName };
2159
+ }
2160
+
2161
+ async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
2162
+ const { syncRules, bucketStorage } = await setupFilteringTest();
2163
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
2164
+ const batch = await test_utils.getBatchArray(
2165
+ bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(BigInt(checkpoint)), [request])
2166
+ );
2167
+ const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
2168
+ return ops;
2169
+ }
2170
+
2171
+ test('case 1: start=5, checkpoint=95 → all ops', async () => {
2172
+ const ops = await getFilteredOps(5, 95);
2173
+ expect(ops).toEqual([10n, 20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
2174
+ });
2175
+
2176
+ test('case 2: start=10, checkpoint=90 → ops in (10,90]', async () => {
2177
+ const ops = await getFilteredOps(10, 90);
2178
+ expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
2179
+ });
2180
+
2181
+ test('case 3: start=15, checkpoint=85 → partial doc boundaries', async () => {
2182
+ const ops = await getFilteredOps(15, 85);
2183
+ expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n]);
2184
+ });
2185
+
2186
+ test('case 4: start=25, checkpoint=55 → spans two docs', async () => {
2187
+ const ops = await getFilteredOps(25, 55);
2188
+ expect(ops).toEqual([30n, 40n, 50n]);
2189
+ });
2190
+
2191
+ test('case 5: start=35, checkpoint=45 → single op within doc', async () => {
2192
+ const ops = await getFilteredOps(35, 45);
2193
+ expect(ops).toEqual([40n]);
2194
+ });
2195
+
2196
+ test('case 6: start=35, checkpoint=65 → full doc B', async () => {
2197
+ const ops = await getFilteredOps(35, 65);
2198
+ expect(ops).toEqual([40n, 50n, 60n]);
2199
+ });
2200
+
2201
+ test('case 7: start=25, checkpoint=35 → single op from doc A', async () => {
2202
+ const ops = await getFilteredOps(25, 35);
2203
+ expect(ops).toEqual([30n]);
2204
+ });
2205
+
2206
+ test('case 8: start=30, checkpoint=40 → op at checkpoint from next doc', async () => {
2207
+ const ops = await getFilteredOps(30, 40);
2208
+ expect(ops).toEqual([40n]);
2209
+ });
2210
+
2211
+ test('case 9: start=100, checkpoint=200 → beyond all docs', async () => {
2212
+ const ops = await getFilteredOps(100, 200);
2213
+ expect(ops).toEqual([]);
2214
+ });
2215
+
2216
+ test('case 10: start=0, checkpoint=5 → before all docs', async () => {
2217
+ const ops = await getFilteredOps(0, 5);
2218
+ expect(ops).toEqual([]);
2219
+ });
2220
+
2221
+ test('case 11: start=50, checkpoint=50 → zero-width range', async () => {
2222
+ const ops = await getFilteredOps(50, 50);
2223
+ expect(ops).toEqual([]);
2224
+ });
2225
+
2226
+ test('case 12: start=45, checkpoint=50 → op at checkpoint boundary', async () => {
2227
+ const ops = await getFilteredOps(45, 50);
2228
+ expect(ops).toEqual([50n]);
2229
+ });
2230
+
2231
+ test('case 13: start=50, checkpoint=55 → no ops strictly after start', async () => {
2232
+ const ops = await getFilteredOps(50, 55);
2233
+ expect(ops).toEqual([]);
2234
+ });
2235
+
2236
+ test('all-filtered first batch still returns data behind the batch boundary', async () => {
2237
+ // Documents straddling the requested (start, end] window are matched by the
2238
+ // query, but contribute no rows after filtering. If an entire server batch
2239
+ // (~101 documents) consists of such straddlers, the remaining documents in
2240
+ // the cursor must still be reachable. Storage reports the straddler buckets
2241
+ // as complete via empty chunks, and the caller re-requests the rest.
2242
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
2243
+ const syncRules = await factory.updateSyncRules(
2244
+ updateSyncRulesFromYaml(
2245
+ `
2246
+ bucket_definitions:
2247
+ by_user:
2248
+ parameters: select request.user_id() as user_id
2249
+ data: [select * from test where owner_id = bucket.user_id]
2250
+ `,
2251
+ { storageVersion }
2252
+ )
2253
+ );
2254
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3;
2255
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
2256
+
2257
+ const start = 5n;
2258
+ const end = 50n;
2259
+
2260
+ // 150 buckets sorted before the data bucket, each with a single document
2261
+ // containing ops at 1 and 100: matched (_id.o=100 > start, min_op=1 <= end),
2262
+ // but no op in (5, 50].
2263
+ const straddlerNames = Array.from({ length: 150 }, (_, i) => `b${`${i}`.padStart(3, '0')}`);
2264
+ const requests = [...straddlerNames, 'zzz'].map((id) =>
2265
+ bucketRequest(syncRules.syncConfigContent[0], `by_user["${id}"]`, start)
2266
+ );
2267
+ const mapping = syncRules.syncConfigContent[0].mapping;
2268
+ const definitionId = mapping.bucketSourceId(requests[0].source);
2269
+ const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
2270
+ const sourceTable = new bson.ObjectId();
2271
+
2272
+ function makeOps(bucket: string, opIds: bigint[]): BucketDataDoc[] {
2273
+ const bucketKey: BucketKey = {
2274
+ replicationStreamId: syncRules.replicationStreamId,
2275
+ definitionId,
2276
+ bucket
2277
+ };
2278
+ return opIds.map((opId) => ({
2279
+ bucketKey,
2280
+ o: opId,
2281
+ op: 'PUT' as const,
2282
+ source_table: sourceTable,
2283
+ source_key: test_utils.rid(`row-${opId}`),
2284
+ table: 'test',
2285
+ row_id: `row-${opId}`,
2286
+ checksum: BigInt(opId) * 10n,
2287
+ data: `{"id":"row-${opId}"}`
2288
+ }));
2289
+ }
2290
+
2291
+ const straddlerDocs = requests
2292
+ .slice(0, -1)
2293
+ .map((request) => serializeBucketData(request.bucket, makeOps(request.bucket, [1n, 100n])));
2294
+ const dataBucket = requests[requests.length - 1].bucket;
2295
+ const dataDoc = serializeBucketData(dataBucket, makeOps(dataBucket, [10n]));
2296
+ await collection.insertMany([...straddlerDocs, dataDoc]);
2297
+
2298
+ // Emulate the caller loop in sync.ts / BucketChecksumState: advance bucket
2299
+ // positions from each chunk, drop completed buckets, and re-request while
2300
+ // any chunk reported has_more.
2301
+ const positions = new Map(requests.map((request) => [request.bucket, request.start]));
2302
+ const pending = new Set(positions.keys());
2303
+ const receivedOps: bigint[] = [];
2304
+ let rounds = 0;
2305
+
2306
+ while (rounds < 10) {
2307
+ rounds++;
2308
+ const roundRequests = requests
2309
+ .filter((request) => pending.has(request.bucket))
2310
+ .map((request) => ({ ...request, start: positions.get(request.bucket)! }));
2311
+ const batch = await test_utils.getBatchArray(
2312
+ bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(end), roundRequests)
2313
+ );
2314
+ let anyHasMore = false;
2315
+ for (const { chunkData } of batch) {
2316
+ positions.set(chunkData.bucket, BigInt(chunkData.next_after));
2317
+ if (chunkData.has_more) {
2318
+ anyHasMore = true;
2319
+ } else {
2320
+ pending.delete(chunkData.bucket);
2321
+ }
2322
+ receivedOps.push(...chunkData.data.map((entry) => BigInt(entry.op_id)));
2323
+ }
2324
+ if (!anyHasMore) {
2325
+ break;
2326
+ }
2327
+ }
2328
+
2329
+ // The op behind the all-straddler first batch must be returned.
2330
+ expect(receivedOps).toEqual([10n]);
2331
+ expect(rounds).toBeLessThan(10);
2332
+ });
2333
+ });
903
2334
  });
904
2335
  }
905
2336
  });