@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +177 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +71 -18
  6. package/dist/storage/MongoBucketStorage.js +516 -202
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
  9. package/dist/storage/implementation/MongoBucketBatch.js +140 -66
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
  15. package/dist/storage/implementation/MongoCompactor.js +87 -481
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
  21. package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +41 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
  47. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  48. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  49. package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
  50. package/dist/storage/implementation/common/PersistedBatch.js +63 -10
  51. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +4 -1
  58. package/dist/storage/implementation/db.js +31 -16
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +55 -13
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  67. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  68. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
  69. package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
  70. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  72. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  73. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  74. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
  75. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
  76. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  77. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  78. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  79. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  80. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  81. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  82. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  83. package/dist/storage/implementation/v1/models.d.ts +10 -1
  84. package/dist/storage/implementation/v1/models.js +1 -0
  85. package/dist/storage/implementation/v1/models.js.map +1 -1
  86. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  87. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  88. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  89. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  90. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  91. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
  93. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
  94. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  96. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  97. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
  99. package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
  100. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
  102. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
  103. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
  105. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
  106. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  107. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
  108. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
  109. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  110. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  111. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  112. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  113. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
  114. package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
  115. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  116. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  117. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  118. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  119. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
  120. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
  121. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  122. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  123. package/dist/storage/implementation/v3/bucket-format.js +79 -0
  124. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  125. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  126. package/dist/storage/implementation/v3/chunking.js +30 -0
  127. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  128. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  129. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  130. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  131. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  132. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  133. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  134. package/dist/storage/implementation/v3/models.d.ts +187 -23
  135. package/dist/storage/implementation/v3/models.js +11 -28
  136. package/dist/storage/implementation/v3/models.js.map +1 -1
  137. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
  138. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
  139. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  140. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
  141. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  142. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  143. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
  144. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
  145. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  146. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
  147. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
  148. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  149. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
  150. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
  151. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  152. package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
  153. package/dist/storage/implementation/v3/source-table-utils.js +206 -0
  154. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  155. package/dist/storage/storage-index.d.ts +5 -4
  156. package/dist/storage/storage-index.js +5 -4
  157. package/dist/storage/storage-index.js.map +1 -1
  158. package/dist/types/types.d.ts +71 -3
  159. package/dist/types/types.js +58 -2
  160. package/dist/types/types.js.map +1 -1
  161. package/dist/utils/test-utils.d.ts +1 -2
  162. package/dist/utils/test-utils.js +7 -1
  163. package/dist/utils/test-utils.js.map +1 -1
  164. package/dist/utils/util.d.ts +9 -2
  165. package/dist/utils/util.js +131 -5
  166. package/dist/utils/util.js.map +1 -1
  167. package/package.json +13 -19
  168. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  169. package/src/storage/MongoBucketStorage.ts +728 -242
  170. package/src/storage/implementation/MongoBucketBatch.ts +196 -91
  171. package/src/storage/implementation/MongoChecksums.ts +93 -153
  172. package/src/storage/implementation/MongoCompactor.ts +106 -631
  173. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  174. package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
  175. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
  176. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  177. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
  178. package/src/storage/implementation/MongoStorageProvider.ts +48 -4
  179. package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
  180. package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
  181. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
  182. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  183. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
  184. package/src/storage/implementation/common/PersistedBatch.ts +83 -15
  185. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  186. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  187. package/src/storage/implementation/db.ts +40 -22
  188. package/src/storage/implementation/models.ts +58 -14
  189. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
  190. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  191. package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
  192. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  193. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
  194. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  195. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  196. package/src/storage/implementation/v1/models.ts +12 -1
  197. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  198. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  199. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
  200. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  201. package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
  202. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
  203. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
  204. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
  205. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  206. package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
  207. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  208. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
  209. package/src/storage/implementation/v3/bucket-format.ts +99 -0
  210. package/src/storage/implementation/v3/chunking.ts +40 -0
  211. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  212. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  213. package/src/storage/implementation/v3/models.ts +228 -58
  214. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
  215. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
  216. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
  217. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
  218. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
  219. package/src/storage/implementation/v3/source-table-utils.ts +398 -0
  220. package/src/storage/storage-index.ts +14 -4
  221. package/src/types/types.ts +76 -2
  222. package/src/utils/test-utils.ts +8 -3
  223. package/src/utils/util.ts +188 -7
  224. package/test/src/__snapshots__/storage.test.ts.snap +4 -54
  225. package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
  226. package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
  227. package/test/src/compact-utils.test.ts +226 -0
  228. package/test/src/helpers/MemoryObjectStorage.ts +61 -0
  229. package/test/src/helpers/s3TestFactory.ts +69 -0
  230. package/test/src/object_storage_lifecycle.test.ts +59 -0
  231. package/test/src/object_storage_usage.test.ts +255 -0
  232. package/test/src/parameter_compacting_v1.test.ts +120 -0
  233. package/test/src/parameter_compacting_v3.test.ts +192 -0
  234. package/test/src/parameter_compaction_fence.test.ts +276 -0
  235. package/test/src/setup.ts +6 -1
  236. package/test/src/storage.test.ts +478 -13
  237. package/test/src/storage_compacting.test.ts +2516 -107
  238. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  239. package/test/src/storage_s3_checksums.test.ts +190 -0
  240. package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
  241. package/test/src/storage_s3_reading.test.ts +768 -0
  242. package/test/src/storage_s3_writing.test.ts +136 -0
  243. package/test/src/storage_sync.test.ts +1507 -76
  244. package/test/src/storeCurrentData.test.ts +227 -0
  245. package/test/src/util.ts +3 -2
  246. package/tsconfig.json +1 -4
  247. package/tsconfig.tsbuildinfo +1 -1
  248. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  249. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  250. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  251. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  252. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  253. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  254. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  255. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  256. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  257. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  258. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  259. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  260. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  261. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  262. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  263. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  264. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  265. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  266. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  267. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  268. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  269. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  270. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  271. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  272. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  273. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  274. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  275. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -0,0 +1,116 @@
1
+ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
2
+ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
3
+ import { bucketRequest, test_utils } from '@powersync/service-core-tests';
4
+ import { describe, expect, test } from 'vitest';
5
+ import { DEFAULT_INLINE_THRESHOLD_BYTES } from '../../src/storage/implementation/common/PersistedBatch.js';
6
+ import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
7
+ import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
8
+ import { env } from './env.js';
9
+ import { MemoryObjectStorage } from './helpers/MemoryObjectStorage.js';
10
+
11
+ const SYNC_RULES_YAML = `
12
+ bucket_definitions:
13
+ global:
14
+ data:
15
+ - SELECT id, description FROM items
16
+ `;
17
+
18
+ function s3Factory(threshold?: number) {
19
+ const memoryStorage = new MemoryObjectStorage();
20
+ const factoryGen = mongoTestStorageFactoryGenerator({
21
+ url: env.MONGO_TEST_URL,
22
+ isCI: env.CI,
23
+ objectStorage: memoryStorage,
24
+ inlineThresholdBytes: threshold
25
+ });
26
+ return { memoryStorage, factoryGen };
27
+ }
28
+
29
+ describe('Object storage inline threshold', () => {
30
+ test('stores small documents inline using the default threshold', async () => {
31
+ const { memoryStorage, factoryGen } = s3Factory();
32
+ await using factory = await factoryGen.factory();
33
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
34
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
35
+ expect(bucketStorage.inlineThresholdBytes).toBe(DEFAULT_INLINE_THRESHOLD_BYTES);
36
+
37
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
38
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
39
+ await writer.markAllSnapshotDone('1/1');
40
+
41
+ await writer.save({
42
+ sourceTable,
43
+ tag: storage.SaveOperationTag.INSERT,
44
+ after: { id: 'small1', description: 'tiny' },
45
+ afterReplicaId: test_utils.rid('small1')
46
+ });
47
+ await writer.save({
48
+ sourceTable,
49
+ tag: storage.SaveOperationTag.INSERT,
50
+ after: { id: 'small2', description: 'tiny' },
51
+ afterReplicaId: test_utils.rid('small2')
52
+ });
53
+ await writer.commit('1/1');
54
+ const checkpoint = await bucketStorage.getCheckpoint();
55
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
56
+
57
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
58
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
59
+ const documents = await db
60
+ .bucketData(bucketStorage.replicationStreamId, definitionId)
61
+ .find({ '_id.b': request.bucket })
62
+ .toArray();
63
+ expect(documents).not.toHaveLength(0);
64
+ for (const document of documents) {
65
+ expect(document.size).toBeLessThanOrEqual(DEFAULT_INLINE_THRESHOLD_BYTES);
66
+ expect(document.ops).toBeDefined();
67
+ expect(document.storage_ref).toBeUndefined();
68
+ }
69
+ expect(memoryStorage.store.size).toBe(0);
70
+
71
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
72
+ const data = test_utils.getBatchData(batch);
73
+ expect(data.map((op) => op.object_id)).toEqual(['small1', 'small2']);
74
+ });
75
+
76
+ test('stores documents above a configured threshold in object storage', async () => {
77
+ const { memoryStorage, factoryGen } = s3Factory(256);
78
+ await using factory = await factoryGen.factory();
79
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
80
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
81
+ expect(bucketStorage.inlineThresholdBytes).toBe(256);
82
+
83
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
84
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
85
+ await writer.markAllSnapshotDone('1/1');
86
+
87
+ await writer.save({
88
+ sourceTable,
89
+ tag: storage.SaveOperationTag.INSERT,
90
+ after: { id: 'large', description: 'value'.repeat(500) },
91
+ afterReplicaId: test_utils.rid('large')
92
+ });
93
+ await writer.commit('1/1');
94
+ const checkpoint = await bucketStorage.getCheckpoint();
95
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
96
+
97
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
98
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
99
+ const documents = await db
100
+ .bucketData(bucketStorage.replicationStreamId, definitionId)
101
+ .find({ '_id.b': request.bucket })
102
+ .toArray();
103
+ expect(documents).not.toHaveLength(0);
104
+ for (const document of documents) {
105
+ expect(document.size).toBeGreaterThan(256);
106
+ expect(document.ops).toBeUndefined();
107
+ expect(document.storage_ref).toBeDefined();
108
+ }
109
+ expect(new Set(memoryStorage.store.keys())).toEqual(
110
+ new Set(documents.map((document) => document.storage_ref!.path))
111
+ );
112
+
113
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
114
+ expect(test_utils.getBatchData(batch).map((op) => op.object_id)).toEqual(['large']);
115
+ });
116
+ });
@@ -0,0 +1,190 @@
1
+ import {
2
+ addChecksums,
3
+ CheckpointChecksumInvalidatedError,
4
+ storage,
5
+ updateSyncRulesFromYaml
6
+ } from '@powersync/service-core';
7
+ import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests';
8
+ import { describe, expect, test } from 'vitest';
9
+ import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
10
+ import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
11
+ import { env } from './env.js';
12
+ import { createS3TestStorageSuite } from './helpers/s3TestFactory.js';
13
+
14
+ const SYNC_RULES_YAML = `
15
+ bucket_definitions:
16
+ global:
17
+ data:
18
+ - SELECT id, description FROM items
19
+ `;
20
+
21
+ function s3Factory() {
22
+ const { objectStorage, factoryGen } = createS3TestStorageSuite({ url: env.MONGO_TEST_URL, isCI: env.CI });
23
+ return { memoryStorage: objectStorage, factoryGen };
24
+ }
25
+
26
+ describe('V3 checksums with S3 object storage', () => {
27
+ test('partial checksum with start straddling S3-backed document', async () => {
28
+ const { factoryGen } = s3Factory();
29
+ await using factory = await factoryGen.factory();
30
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
31
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
32
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
33
+
34
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
35
+ const bucket = request.bucket;
36
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
37
+
38
+ // Write 6 ops through S3 writer — all land in one S3-backed doc
39
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
40
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
41
+ await writer.markAllSnapshotDone('1/1');
42
+ for (const id of ['A', 'B', 'C', 'D', 'E', 'F']) {
43
+ await writer.save({
44
+ sourceTable,
45
+ tag: storage.SaveOperationTag.INSERT,
46
+ after: { id, description: id },
47
+ afterReplicaId: test_utils.rid(id)
48
+ });
49
+ }
50
+ await writer.commit('1/1');
51
+ const checkpoint = await bucketStorage.getCheckpoint();
52
+
53
+ // Baseline: full checksum with no compacted_state
54
+ const full = (await bucketStorage.getChecksums(checkpoint, [request])).get(bucket)!;
55
+
56
+ // Set compacted_state.op_id = 3 to create a partial range starting after op 3.
57
+ // The doc has min_op=1, _id.o=6 so is_fully_included=false for the range (3, 6].
58
+ await db.bucketState(bucketStorage.replicationStreamId).updateOne(
59
+ { _id: { d: definitionId, b: bucket } },
60
+ {
61
+ $set: {
62
+ last_op: 3n,
63
+ compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: 0n, chunks: 0, at: new Date() }
64
+ }
65
+ },
66
+ { upsert: true }
67
+ );
68
+
69
+ const partial = (await bucketStorage.getChecksums(checkpoint, [request])).get(bucket)!;
70
+ expect(partial.checksum).toBe(full.checksum);
71
+ expect(partial.count).toBe(full.count);
72
+ });
73
+
74
+ test('partial checksum with end straddling S3-backed document invalidates the checkpoint', async () => {
75
+ const { factoryGen } = s3Factory();
76
+ await using factory = await factoryGen.factory();
77
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
78
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
79
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
80
+
81
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
82
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
83
+
84
+ // Write several operations into one S3-backed document that straddles the
85
+ // requested checkpoint.
86
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
87
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
88
+ await writer.markAllSnapshotDone('1/1');
89
+ for (let i = 1; i <= 12; i++) {
90
+ const id = `row${i}`;
91
+ await writer.save({
92
+ sourceTable,
93
+ tag: storage.SaveOperationTag.INSERT,
94
+ after: { id, description: `value${i}`.repeat(50) },
95
+ afterReplicaId: test_utils.rid(id)
96
+ });
97
+ }
98
+ await writer.commit('1/1');
99
+ const documents = await db
100
+ .bucketData(bucketStorage.replicationStreamId, definitionId)
101
+ .find({ '_id.b': request.bucket })
102
+ .toArray();
103
+ expect(documents).toHaveLength(1);
104
+ expect(documents[0].storage_ref).toBeDefined();
105
+ expect(documents[0].ops).toBeUndefined();
106
+
107
+ // The document starts before op 7 and ends after it.
108
+ await expect(bucketStorage.getChecksums(test_utils.testCheckpoint(7n), [request])).rejects.toBeInstanceOf(
109
+ CheckpointChecksumInvalidatedError
110
+ );
111
+ });
112
+
113
+ test('checksum preserved after CLEAR-producing S3 compaction', async () => {
114
+ const { factoryGen } = s3Factory();
115
+ await using factory = await factoryGen.factory();
116
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
117
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
118
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
119
+
120
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
121
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
122
+
123
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
124
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
125
+ await writer.markAllSnapshotDone('1/1');
126
+
127
+ // A@1, A@2 get superseded by A@4 (same replica_id for dedup). B@3 independent.
128
+ await writer.save({
129
+ sourceTable,
130
+ tag: storage.SaveOperationTag.INSERT,
131
+ after: { id: 'A', description: 'v1' },
132
+ afterReplicaId: test_utils.rid('A')
133
+ });
134
+ await writer.save({
135
+ sourceTable,
136
+ tag: storage.SaveOperationTag.UPDATE,
137
+ after: { id: 'A', description: 'v2' },
138
+ afterReplicaId: test_utils.rid('A')
139
+ });
140
+ await writer.save({
141
+ sourceTable,
142
+ tag: storage.SaveOperationTag.INSERT,
143
+ after: { id: 'B', description: 'beta' },
144
+ afterReplicaId: test_utils.rid('B')
145
+ });
146
+ await writer.save({
147
+ sourceTable,
148
+ tag: storage.SaveOperationTag.UPDATE,
149
+ after: { id: 'A', description: 'v4' },
150
+ afterReplicaId: test_utils.rid('A')
151
+ });
152
+ await writer.commit('1/1');
153
+ const checkpoint = await bucketStorage.getCheckpoint();
154
+
155
+ // Compact — produces CLEAR doc from collapsed MOVEs
156
+ await compactActive(factory, {
157
+ maxOpId: checkpoint.checkpoint,
158
+ compactBuckets: [request.bucket],
159
+ clearBatchLimit: 200,
160
+ moveBatchLimit: 10,
161
+ moveBatchQueryLimit: 10,
162
+ moveBatchByteLimit: 1024,
163
+ minBucketChanges: 1,
164
+ minChangeRatio: 0
165
+ });
166
+
167
+ // Shift compacted_state.op_id back to before the CLEAR doc so getChecksums
168
+ // queries the pipeline (not just reads compacted_state directly).
169
+ await db
170
+ .bucketState(bucketStorage.replicationStreamId)
171
+ .updateOne({ _id: { d: definitionId, b: request.bucket } }, { $set: { 'compacted_state.op_id': 0n } });
172
+
173
+ // Ground truth: sum of doc-level checksum fields
174
+ const docs = await db
175
+ .bucketData(bucketStorage.replicationStreamId, definitionId)
176
+ .find({ '_id.b': request.bucket })
177
+ .toArray();
178
+ const groundTruth = docs.reduce((sum: number, d: any) => addChecksums(sum, Number(d.checksum)), 0);
179
+ expect(docs.some((doc) => doc.has_clear_op)).toBe(true);
180
+ expect(docs.every((doc) => doc.storage_ref != null && doc.ops == null)).toBe(true);
181
+
182
+ const checksum = (await bucketStorage.getChecksums(checkpoint, [request])).get(request.bucket)!;
183
+ expect(checksum.checksum).toBe(groundTruth);
184
+
185
+ // The two superseded A operations collapse into CLEAR.
186
+ const batchAfter = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
187
+ const dataAfter = batchAfter.flatMap((chunk) => chunk.chunkData.data);
188
+ expect(dataAfter).toMatchObject([{ op: 'CLEAR' }, { object_id: 'B', op: 'PUT' }, { object_id: 'A', op: 'PUT' }]);
189
+ });
190
+ });