@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +177 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +71 -18
  6. package/dist/storage/MongoBucketStorage.js +516 -202
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
  9. package/dist/storage/implementation/MongoBucketBatch.js +140 -66
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
  15. package/dist/storage/implementation/MongoCompactor.js +87 -481
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
  21. package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +41 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
  47. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  48. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  49. package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
  50. package/dist/storage/implementation/common/PersistedBatch.js +63 -10
  51. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +4 -1
  58. package/dist/storage/implementation/db.js +31 -16
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +55 -13
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  67. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  68. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
  69. package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
  70. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  72. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  73. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  74. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
  75. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
  76. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  77. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  78. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  79. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  80. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  81. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  82. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  83. package/dist/storage/implementation/v1/models.d.ts +10 -1
  84. package/dist/storage/implementation/v1/models.js +1 -0
  85. package/dist/storage/implementation/v1/models.js.map +1 -1
  86. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  87. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  88. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  89. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  90. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  91. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
  93. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
  94. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  96. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  97. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
  99. package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
  100. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
  102. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
  103. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
  105. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
  106. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  107. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
  108. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
  109. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  110. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  111. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  112. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  113. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
  114. package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
  115. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  116. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  117. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  118. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  119. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
  120. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
  121. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  122. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  123. package/dist/storage/implementation/v3/bucket-format.js +79 -0
  124. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  125. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  126. package/dist/storage/implementation/v3/chunking.js +30 -0
  127. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  128. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  129. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  130. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  131. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  132. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  133. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  134. package/dist/storage/implementation/v3/models.d.ts +187 -23
  135. package/dist/storage/implementation/v3/models.js +11 -28
  136. package/dist/storage/implementation/v3/models.js.map +1 -1
  137. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
  138. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
  139. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  140. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
  141. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  142. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  143. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
  144. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
  145. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  146. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
  147. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
  148. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  149. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
  150. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
  151. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  152. package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
  153. package/dist/storage/implementation/v3/source-table-utils.js +206 -0
  154. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  155. package/dist/storage/storage-index.d.ts +5 -4
  156. package/dist/storage/storage-index.js +5 -4
  157. package/dist/storage/storage-index.js.map +1 -1
  158. package/dist/types/types.d.ts +71 -3
  159. package/dist/types/types.js +58 -2
  160. package/dist/types/types.js.map +1 -1
  161. package/dist/utils/test-utils.d.ts +1 -2
  162. package/dist/utils/test-utils.js +7 -1
  163. package/dist/utils/test-utils.js.map +1 -1
  164. package/dist/utils/util.d.ts +9 -2
  165. package/dist/utils/util.js +131 -5
  166. package/dist/utils/util.js.map +1 -1
  167. package/package.json +13 -19
  168. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  169. package/src/storage/MongoBucketStorage.ts +728 -242
  170. package/src/storage/implementation/MongoBucketBatch.ts +196 -91
  171. package/src/storage/implementation/MongoChecksums.ts +93 -153
  172. package/src/storage/implementation/MongoCompactor.ts +106 -631
  173. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  174. package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
  175. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
  176. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  177. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
  178. package/src/storage/implementation/MongoStorageProvider.ts +48 -4
  179. package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
  180. package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
  181. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
  182. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  183. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
  184. package/src/storage/implementation/common/PersistedBatch.ts +83 -15
  185. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  186. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  187. package/src/storage/implementation/db.ts +40 -22
  188. package/src/storage/implementation/models.ts +58 -14
  189. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
  190. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  191. package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
  192. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  193. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
  194. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  195. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  196. package/src/storage/implementation/v1/models.ts +12 -1
  197. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  198. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  199. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
  200. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  201. package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
  202. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
  203. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
  204. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
  205. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  206. package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
  207. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  208. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
  209. package/src/storage/implementation/v3/bucket-format.ts +99 -0
  210. package/src/storage/implementation/v3/chunking.ts +40 -0
  211. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  212. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  213. package/src/storage/implementation/v3/models.ts +228 -58
  214. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
  215. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
  216. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
  217. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
  218. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
  219. package/src/storage/implementation/v3/source-table-utils.ts +398 -0
  220. package/src/storage/storage-index.ts +14 -4
  221. package/src/types/types.ts +76 -2
  222. package/src/utils/test-utils.ts +8 -3
  223. package/src/utils/util.ts +188 -7
  224. package/test/src/__snapshots__/storage.test.ts.snap +4 -54
  225. package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
  226. package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
  227. package/test/src/compact-utils.test.ts +226 -0
  228. package/test/src/helpers/MemoryObjectStorage.ts +61 -0
  229. package/test/src/helpers/s3TestFactory.ts +69 -0
  230. package/test/src/object_storage_lifecycle.test.ts +59 -0
  231. package/test/src/object_storage_usage.test.ts +255 -0
  232. package/test/src/parameter_compacting_v1.test.ts +120 -0
  233. package/test/src/parameter_compacting_v3.test.ts +192 -0
  234. package/test/src/parameter_compaction_fence.test.ts +276 -0
  235. package/test/src/setup.ts +6 -1
  236. package/test/src/storage.test.ts +478 -13
  237. package/test/src/storage_compacting.test.ts +2516 -107
  238. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  239. package/test/src/storage_s3_checksums.test.ts +190 -0
  240. package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
  241. package/test/src/storage_s3_reading.test.ts +768 -0
  242. package/test/src/storage_s3_writing.test.ts +136 -0
  243. package/test/src/storage_sync.test.ts +1507 -76
  244. package/test/src/storeCurrentData.test.ts +227 -0
  245. package/test/src/util.ts +3 -2
  246. package/tsconfig.json +1 -4
  247. package/tsconfig.tsbuildinfo +1 -1
  248. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  249. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  250. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  251. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  252. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  253. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  254. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  255. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  256. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  257. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  258. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  259. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  260. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  261. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  262. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  263. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  264. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  265. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  266. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  267. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  268. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  269. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  270. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  271. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  272. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  273. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  274. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  275. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -0,0 +1,543 @@
1
+ import * as lib_mongo from '@powersync/lib-service-mongodb';
2
+ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
3
+ import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests';
4
+ import { describe, expect, test } from 'vitest';
5
+ import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
6
+ import { MongoSyncBucketStorageV3 } from '../../src/storage/implementation/v3/MongoSyncBucketStorageV3.js';
7
+ import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
8
+ import { ObjectStorageError } from '../../src/storage/implementation/v3/object-storage/ObjectStorage.js';
9
+ import { env } from './env.js';
10
+ import { createMemoryS3TestStorageSuite, createS3TestStorageSuite } from './helpers/s3TestFactory.js';
11
+
12
+ const SYNC_RULES_YAML = `
13
+ bucket_definitions:
14
+ global:
15
+ data:
16
+ - SELECT id, description FROM items
17
+ `;
18
+
19
+ function s3Factory() {
20
+ const { objectStorage, factoryGen } = createS3TestStorageSuite({ url: env.MONGO_TEST_URL, isCI: env.CI });
21
+ return { memoryStorage: objectStorage, factory: factoryGen };
22
+ }
23
+
24
+ function memoryS3Factory(options: { inlineThresholdBytes?: number } = {}) {
25
+ const { objectStorage, factoryGen } = createMemoryS3TestStorageSuite({
26
+ url: env.MONGO_TEST_URL,
27
+ isCI: env.CI,
28
+ inlineThresholdBytes: options.inlineThresholdBytes ?? 0
29
+ });
30
+ return { memoryStorage: objectStorage, factory: factoryGen };
31
+ }
32
+
33
+ async function expectUsageMatchesBucketData(bucketStorage: MongoSyncBucketStorage, definitionId: string) {
34
+ const v3BucketStorage = bucketStorage as MongoSyncBucketStorageV3;
35
+ const db = v3BucketStorage.db as VersionedPowerSyncMongoV3;
36
+ const documents = await db.bucketData(v3BucketStorage.replicationStreamId, definitionId).find({}).toArray();
37
+ const expectedBytes = documents.reduce((sum, document) => sum + BigInt(document.storage_ref?.file_size ?? 0), 0n);
38
+ const entries = await db.objectStorageUsage
39
+ .aggregate<{ active_bytes: bigint }>([
40
+ { $match: { '_id.g': v3BucketStorage.replicationStreamId } },
41
+ { $project: { definitions: { $objectToArray: '$definitions' } } },
42
+ { $unwind: '$definitions' },
43
+ { $group: { _id: null, active_bytes: { $sum: '$definitions.v' } } }
44
+ ])
45
+ .toArray()
46
+ .catch((error) => {
47
+ if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
48
+ return [];
49
+ }
50
+ throw error;
51
+ });
52
+ expect(BigInt(entries[0]?.active_bytes ?? 0)).toBe(expectedBytes);
53
+ }
54
+
55
+ describe('S3 compaction storage lifecycle', () => {
56
+ test('a later compaction recovers from a transient object storage failure', async () => {
57
+ const { memoryStorage, factory: factoryGen } = memoryS3Factory();
58
+ await using factory = await factoryGen.factory();
59
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
60
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
61
+
62
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
63
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
64
+ await writer.markAllSnapshotDone('1/1');
65
+ for (const [index, description] of ['old', 'new'].entries()) {
66
+ await writer.save({
67
+ sourceTable,
68
+ tag: index === 0 ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE,
69
+ after: { id: 'A', description },
70
+ afterReplicaId: test_utils.rid('A')
71
+ });
72
+ await writer.commit(`${index + 1}/1`);
73
+ }
74
+
75
+ const originalGet = memoryStorage.get.bind(memoryStorage);
76
+ let injectedFailure = false;
77
+ memoryStorage.get = async (...args) => {
78
+ if (!injectedFailure) {
79
+ injectedFailure = true;
80
+ throw new ObjectStorageError('temporary object storage failure', {
81
+ cause: new Error('socket reset'),
82
+ retryable: true
83
+ });
84
+ }
85
+ return originalGet(...args);
86
+ };
87
+
88
+ const checkpoint = await bucketStorage.getCheckpoint();
89
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
90
+ const compactOptions = {
91
+ maxOpId: checkpoint.checkpoint,
92
+ compactBuckets: [request.bucket],
93
+ minBucketChanges: 1,
94
+ minChangeRatio: 0
95
+ };
96
+ await expect(compactActive(factory, compactOptions)).rejects.toThrow('temporary object storage failure');
97
+ await compactActive(factory, compactOptions);
98
+
99
+ expect(injectedFailure).toBe(true);
100
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
101
+ const data = batch.flatMap((chunk) => chunk.chunkData.data);
102
+ expect(data).toHaveLength(2);
103
+ });
104
+
105
+ test('small inline updates merge into an inline replacement', async () => {
106
+ const { memoryStorage, factory: factoryGen } = memoryS3Factory({ inlineThresholdBytes: 10_000 });
107
+ await using factory = await factoryGen.factory();
108
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
109
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
110
+
111
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
112
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 10);
113
+ await writer.markAllSnapshotDone('1/1');
114
+ await writer.save({
115
+ sourceTable,
116
+ tag: storage.SaveOperationTag.INSERT,
117
+ after: { id: 'A', description: 'old' },
118
+ afterReplicaId: test_utils.rid('A')
119
+ });
120
+ await writer.commit('1/1');
121
+ await writer.save({
122
+ sourceTable,
123
+ tag: storage.SaveOperationTag.UPDATE,
124
+ after: { id: 'A', description: 'new' },
125
+ afterReplicaId: test_utils.rid('A')
126
+ });
127
+ await writer.commit('2/1');
128
+
129
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
130
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
131
+ const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
132
+ expect(await collection.countDocuments({ storage_ref: { $exists: true } })).toBe(0);
133
+
134
+ const checkpoint = await bucketStorage.getCheckpoint();
135
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
136
+ await compactActive(factory, {
137
+ maxOpId: checkpoint.checkpoint,
138
+ compactBuckets: [request.bucket],
139
+ minBucketChanges: 1,
140
+ minChangeRatio: 0
141
+ });
142
+
143
+ const docs = await collection.find({}).toArray();
144
+ expect(docs).toHaveLength(1);
145
+ expect(docs[0].storage_ref).toBeUndefined();
146
+ expect(docs[0].ops).toBeDefined();
147
+ expect(memoryStorage.store.size).toBe(0);
148
+ await expectUsageMatchesBucketData(bucketStorage, definitionId);
149
+ });
150
+
151
+ test('boundary CLEAR replacement updates active object-storage usage', async () => {
152
+ const { factory: factoryGen } = memoryS3Factory();
153
+ await using factory = await factoryGen.factory();
154
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
155
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
156
+
157
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
158
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 11);
159
+ await writer.markAllSnapshotDone('1/1');
160
+ for (const [index, description] of ['first', 'second', 'latest'].entries()) {
161
+ await writer.save({
162
+ sourceTable,
163
+ tag: index === 0 ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE,
164
+ after: { id: 'A', description: description.repeat(100) },
165
+ afterReplicaId: test_utils.rid('A')
166
+ });
167
+ }
168
+ await writer.commit('1/1');
169
+
170
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
171
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
172
+ const checkpoint = await bucketStorage.getCheckpoint();
173
+ await compactActive(factory, {
174
+ maxOpId: checkpoint.checkpoint,
175
+ compactBuckets: [request.bucket],
176
+ minBucketChanges: 1,
177
+ minChangeRatio: 0
178
+ });
179
+
180
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
181
+ const data = batch.flatMap((chunk) => chunk.chunkData.data);
182
+ expect(data.some((op: any) => op.op === 'CLEAR')).toBe(true);
183
+ await expectUsageMatchesBucketData(bucketStorage, definitionId);
184
+ });
185
+
186
+ test('leading and boundary CLEAR replacements update active object-storage usage', async () => {
187
+ const { factory: factoryGen } = memoryS3Factory();
188
+ await using factory = await factoryGen.factory();
189
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
190
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
191
+
192
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
193
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 12);
194
+ await writer.markAllSnapshotDone('1/1');
195
+
196
+ // The first and last A versions force a leading MOVE document. The
197
+ // middle document also contains a large surviving PUT, keeping it as the
198
+ // CLEAR boundary document rather than merging it with the leading group.
199
+ await writer.save({
200
+ sourceTable,
201
+ tag: storage.SaveOperationTag.INSERT,
202
+ after: { id: 'A', description: 'old' },
203
+ afterReplicaId: test_utils.rid('A')
204
+ });
205
+ await writer.commit('1/1');
206
+
207
+ await writer.save({
208
+ sourceTable,
209
+ tag: storage.SaveOperationTag.UPDATE,
210
+ after: { id: 'A', description: 'middle' },
211
+ afterReplicaId: test_utils.rid('A')
212
+ });
213
+ await writer.save({
214
+ sourceTable,
215
+ tag: storage.SaveOperationTag.INSERT,
216
+ after: { id: 'B', description: 'b'.repeat(990_000) },
217
+ afterReplicaId: test_utils.rid('B')
218
+ });
219
+ await writer.commit('2/1');
220
+
221
+ await writer.save({
222
+ sourceTable,
223
+ tag: storage.SaveOperationTag.UPDATE,
224
+ after: { id: 'A', description: 'latest'.repeat(10_000) },
225
+ afterReplicaId: test_utils.rid('A')
226
+ });
227
+ await writer.commit('3/1');
228
+
229
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
230
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
231
+ const checkpoint = await bucketStorage.getCheckpoint();
232
+ await compactActive(factory, {
233
+ maxOpId: checkpoint.checkpoint,
234
+ compactBuckets: [request.bucket],
235
+ clearBatchLimit: 2,
236
+ moveBatchQueryLimit: 1,
237
+ minBucketChanges: 1,
238
+ minChangeRatio: 0
239
+ });
240
+
241
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
242
+ const data = batch.flatMap((chunk) => chunk.chunkData.data);
243
+ expect(data.some((op: any) => op.op === 'CLEAR')).toBe(true);
244
+ await expectUsageMatchesBucketData(bucketStorage, definitionId);
245
+ });
246
+
247
+ test('MOVE compaction merges adjacent objects using their compacted size in one write', async () => {
248
+ const { memoryStorage, factory: factoryGen } = memoryS3Factory();
249
+ await using factory = await factoryGen.factory();
250
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
251
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
252
+
253
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
254
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
255
+ await writer.markAllSnapshotDone('1/1');
256
+
257
+ await writer.save({
258
+ sourceTable,
259
+ tag: storage.SaveOperationTag.INSERT,
260
+ after: { id: 'A', description: 'old'.repeat(120_000) },
261
+ afterReplicaId: test_utils.rid('A')
262
+ });
263
+ await writer.commit('1/1');
264
+
265
+ await writer.save({
266
+ sourceTable,
267
+ tag: storage.SaveOperationTag.INSERT,
268
+ after: { id: 'B', description: 'middle'.repeat(60_000) },
269
+ afterReplicaId: test_utils.rid('B')
270
+ });
271
+ await writer.commit('2/1');
272
+
273
+ await writer.save({
274
+ sourceTable,
275
+ tag: storage.SaveOperationTag.UPDATE,
276
+ after: { id: 'A', description: 'new'.repeat(120_000) },
277
+ afterReplicaId: test_utils.rid('A')
278
+ });
279
+ await writer.commit('3/1');
280
+
281
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
282
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
283
+ const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
284
+ const docsBefore = await collection.find({}).sort({ _id: 1 }).toArray();
285
+ expect(docsBefore).toHaveLength(3);
286
+
287
+ const oldPaths = new Set(docsBefore.map((doc) => doc.storage_ref!.path));
288
+ const objectStore = memoryStorage.store;
289
+ expect(objectStore.size).toBe(3);
290
+
291
+ const checkpoint = await bucketStorage.getCheckpoint();
292
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
293
+ await compactActive(factory, {
294
+ maxOpId: checkpoint.checkpoint,
295
+ compactBuckets: [request.bucket],
296
+ clearBatchLimit: 200,
297
+ moveBatchLimit: 10,
298
+ // Force each input object into a different MongoDB query batch. The
299
+ // pending merge group must survive across those batches.
300
+ moveBatchQueryLimit: 1,
301
+ moveBatchByteLimit: 16 * 1024 * 1024,
302
+ minBucketChanges: 1,
303
+ minChangeRatio: 0
304
+ });
305
+
306
+ const docsAfter = await collection.find({}).toArray();
307
+ expect(docsAfter).toHaveLength(1);
308
+ expect(docsAfter[0]).toMatchObject({
309
+ min_op: docsBefore[0].min_op,
310
+ _id: { o: docsBefore[2]._id.o },
311
+ count: 3
312
+ });
313
+ expect(oldPaths.has(docsAfter[0].storage_ref!.path)).toBe(false);
314
+
315
+ // The three old objects remain during their grace period and exactly one
316
+ // final replacement was uploaded. There was no intermediate MOVE-only
317
+ // object before the merge.
318
+ expect(objectStore.size).toBe(4);
319
+ await expectUsageMatchesBucketData(bucketStorage, definitionId);
320
+ });
321
+
322
+ test('compaction only retires S3 objects at or below the maxOpId horizon', async () => {
323
+ const { memoryStorage, factory: factoryGen } = memoryS3Factory();
324
+ await using factory = await factoryGen.factory();
325
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
326
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
327
+
328
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
329
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
330
+ await writer.markAllSnapshotDone('1/1');
331
+
332
+ // Commit each operation separately to create objects on both sides of the
333
+ // compaction horizon.
334
+ for (let i = 1; i <= 12; i++) {
335
+ const letter = String.fromCharCode(64 + i); // A..L
336
+ await writer.save({
337
+ sourceTable,
338
+ tag: storage.SaveOperationTag.INSERT,
339
+ after: { id: letter, description: `val${i}`.repeat(50) },
340
+ afterReplicaId: test_utils.rid(letter)
341
+ });
342
+ await writer.commit(`${i}/1`);
343
+ }
344
+
345
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
346
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
347
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
348
+ const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
349
+ const docsBefore = await collection.find({}).sort({ '_id.o': 1 }).toArray();
350
+ expect(docsBefore).toHaveLength(12);
351
+
352
+ const maxOpId = docsBefore[5]._id.o;
353
+ const readCheckpoint = test_utils.testCheckpoint(docsBefore[docsBefore.length - 1]._id.o);
354
+ const lowerPaths = new Set(docsBefore.slice(0, 6).map((doc) => doc.storage_ref!.path));
355
+ const upperPaths = new Set(docsBefore.slice(6).map((doc) => doc.storage_ref!.path));
356
+
357
+ await compactActive(factory, {
358
+ maxOpId,
359
+ compactBuckets: [request.bucket],
360
+ clearBatchLimit: 200,
361
+ moveBatchLimit: 10,
362
+ moveBatchQueryLimit: 10,
363
+ moveBatchByteLimit: 1024,
364
+ minBucketChanges: 1,
365
+ minChangeRatio: 0
366
+ });
367
+
368
+ const docsAfter = await collection.find({}).sort({ '_id.o': 1 }).toArray();
369
+ const referencedPathsAfter = new Set(docsAfter.map((doc) => doc.storage_ref!.path));
370
+
371
+ // The lower documents are replaced and queued for delayed deletion.
372
+ expect([...lowerPaths].some((path) => referencedPathsAfter.has(path))).toBe(false);
373
+ const pendingDeletes = await db.pendingObjectStorageDeletes(bucketStorage.replicationStreamId).find({}).toArray();
374
+ const retiredPaths = new Set(pendingDeletes.map((marker) => marker.path));
375
+ expect([...lowerPaths].every((path) => retiredPaths.has(path))).toBe(true);
376
+
377
+ // Objects above the horizon remain referenced and must not be retired.
378
+ expect([...upperPaths].every((path) => referencedPathsAfter.has(path))).toBe(true);
379
+ expect([...upperPaths].some((path) => retiredPaths.has(path))).toBe(false);
380
+ expect([...upperPaths].every((path) => memoryStorage.store.has(path))).toBe(true);
381
+
382
+ // Retired objects remain readable during the reference grace period.
383
+ expect([...lowerPaths].every((path) => memoryStorage.store.has(path))).toBe(true);
384
+ await expectUsageMatchesBucketData(bucketStorage, definitionId);
385
+
386
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(readCheckpoint, [request]));
387
+ const data = batch.flatMap((chunk) => chunk.chunkData.data);
388
+ expect(data).toHaveLength(12);
389
+ for (let i = 7; i <= 12; i++) {
390
+ const letter = String.fromCharCode(64 + i);
391
+ expect(data.find((op: any) => op.object_id === letter)).toMatchObject({ op: 'PUT' });
392
+ }
393
+ });
394
+
395
+ test('compaction round-trip hydrates and replaces S3-backed data', async () => {
396
+ const { memoryStorage, factory: factoryGen } = memoryS3Factory();
397
+ await using factory = await factoryGen.factory();
398
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
399
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
400
+
401
+ // Write several operations, including a repeated object id, to exercise
402
+ // the compaction round-trip.
403
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
404
+ const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
405
+ await writer.markAllSnapshotDone('1/1');
406
+
407
+ // Op 1: A@first
408
+ await writer.save({
409
+ sourceTable,
410
+ tag: storage.SaveOperationTag.INSERT,
411
+ after: { id: 'A', description: 'first version' },
412
+ afterReplicaId: test_utils.rid('A')
413
+ });
414
+
415
+ // Op 2: B
416
+ await writer.save({
417
+ sourceTable,
418
+ tag: storage.SaveOperationTag.INSERT,
419
+ after: { id: 'B', description: 'second item' },
420
+ afterReplicaId: test_utils.rid('B')
421
+ });
422
+
423
+ // Op 3: A@updated (supersedes Op 1)
424
+ await writer.save({
425
+ sourceTable,
426
+ tag: storage.SaveOperationTag.UPDATE,
427
+ after: { id: 'A', description: 'updated version' },
428
+ afterReplicaId: test_utils.rid('A')
429
+ });
430
+
431
+ // Op 4: C
432
+ await writer.save({
433
+ sourceTable,
434
+ tag: storage.SaveOperationTag.INSERT,
435
+ after: { id: 'C', description: 'third item' },
436
+ afterReplicaId: test_utils.rid('C')
437
+ });
438
+
439
+ await writer.commit('1/1');
440
+
441
+ // Verify S3 objects were created (write path works).
442
+ const storedPaths = memoryStorage.store;
443
+ expect(storedPaths.size).toBeGreaterThan(0);
444
+
445
+ // Verify MongoDB documents have storage_ref (no ops[]).
446
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
447
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
448
+ const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
449
+ const bucketStateCollection = db.bucketState(bucketStorage.replicationStreamId);
450
+
451
+ const docsBefore = await collection.find({}).toArray();
452
+ expect(docsBefore).toHaveLength(1);
453
+ for (const doc of docsBefore) {
454
+ expect(doc.storage_ref).toBeDefined();
455
+ expect(doc.ops).toBeUndefined();
456
+ }
457
+
458
+ // Get checkpoint and the bucket name.
459
+ const checkpoint = await bucketStorage.getCheckpoint();
460
+ expect(checkpoint.checkpoint).toBeGreaterThan(0n);
461
+
462
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
463
+ const bucket = request.bucket;
464
+
465
+ // Read bucket_state before compaction to confirm the writer recorded its
466
+ // aggregate statistics and scheduled a compact check.
467
+ const bucketStateBefore = await bucketStateCollection.findOne({
468
+ _id: { d: definitionId, b: bucket }
469
+ });
470
+ expect(bucketStateBefore).toBeDefined();
471
+ expect(bucketStateBefore!.bucket_stats.count).toBeGreaterThan(0);
472
+ expect(bucketStateBefore!.next_compact_check).toBeDefined();
473
+
474
+ // Record the input operations and object path.
475
+ const batchBefore = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
476
+ const dataBefore = test_utils.getBatchData(batchBefore);
477
+ expect(dataBefore).toHaveLength(4);
478
+ const oldS3Paths = new Set(docsBefore.map((doc) => doc.storage_ref!.path));
479
+ const expectedCompactedOpId = docsBefore
480
+ .filter((doc) => doc._id.b === bucket && doc._id.o <= checkpoint.checkpoint)
481
+ .reduce<bigint | null>((highest, doc) => (highest == null || doc._id.o > highest ? doc._id.o : highest), null);
482
+ expect(expectedCompactedOpId).not.toBeNull();
483
+
484
+ // Compact the bucket.
485
+ await compactActive(factory, {
486
+ maxOpId: checkpoint.checkpoint,
487
+ compactBuckets: [bucket],
488
+ clearBatchLimit: 200,
489
+ moveBatchLimit: 10,
490
+ moveBatchQueryLimit: 10,
491
+ moveBatchByteLimit: 1024,
492
+ minBucketChanges: 1,
493
+ minChangeRatio: 0
494
+ });
495
+
496
+ // The compacted state reflects the hydrated object contents.
497
+ const bucketStateAfter = await bucketStateCollection.findOne({
498
+ _id: { d: definitionId, b: bucket }
499
+ });
500
+ expect(bucketStateAfter).toBeDefined();
501
+ expect(bucketStateAfter!.compacted_state).toBeDefined();
502
+
503
+ expect(bucketStateAfter!.compacted_state!.checksum).not.toBe(0n);
504
+ expect(bucketStateAfter!.compacted_state!.count).toBe(4);
505
+
506
+ // Record the highest persisted document that was actually included in the
507
+ // compaction scan, rather than the requested upper bound.
508
+ expect(bucketStateAfter!.compacted_state!.op_id).toBe(expectedCompactedOpId);
509
+
510
+ // Compacted MongoDB documents remain metadata shells for object storage.
511
+ const docsAfter = await collection.find({}).toArray();
512
+ expect(docsAfter).toHaveLength(1);
513
+ for (const doc of docsAfter) {
514
+ expect(doc.storage_ref).toBeDefined();
515
+ expect(doc.ops).toBeUndefined();
516
+ }
517
+
518
+ // Paths are globally unique, and compaction records delayed deletion markers
519
+ // instead of deleting objects while old readers may still be downloading them.
520
+ const afterS3Paths = new Set(docsAfter.map((doc) => doc.storage_ref!.path));
521
+ expect([...afterS3Paths].some((path) => oldS3Paths.has(path))).toBe(false);
522
+ const pendingDeletes = await db.pendingObjectStorageDeletes(bucketStorage.replicationStreamId).find({}).toArray();
523
+ expect(pendingDeletes.map((marker) => marker.path)).toEqual(expect.arrayContaining([...oldS3Paths]));
524
+ for (const path of oldS3Paths) {
525
+ expect(storedPaths.has(path)).toBe(true);
526
+ }
527
+ for (const path of afterS3Paths) {
528
+ expect(storedPaths.has(path)).toBe(true);
529
+ }
530
+ await expectUsageMatchesBucketData(bucketStorage, definitionId);
531
+
532
+ // The replacement object is readable and contains the expected compacted
533
+ // operation sequence.
534
+ const batchAfter = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
535
+ const dataAfter = batchAfter.flatMap((chunk) => chunk.chunkData.data);
536
+ expect(dataAfter).toMatchObject([
537
+ { op: 'MOVE' },
538
+ { object_id: 'B', op: 'PUT' },
539
+ { object_id: 'A', op: 'PUT', data: '{"id":"A","description":"updated version"}' },
540
+ { object_id: 'C', op: 'PUT' }
541
+ ]);
542
+ });
543
+ });