@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +177 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +71 -18
  6. package/dist/storage/MongoBucketStorage.js +516 -202
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
  9. package/dist/storage/implementation/MongoBucketBatch.js +140 -66
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
  15. package/dist/storage/implementation/MongoCompactor.js +87 -481
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
  21. package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +41 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
  47. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  48. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  49. package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
  50. package/dist/storage/implementation/common/PersistedBatch.js +63 -10
  51. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +4 -1
  58. package/dist/storage/implementation/db.js +31 -16
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +55 -13
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  67. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  68. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
  69. package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
  70. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  72. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  73. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  74. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
  75. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
  76. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  77. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  78. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  79. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  80. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  81. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  82. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  83. package/dist/storage/implementation/v1/models.d.ts +10 -1
  84. package/dist/storage/implementation/v1/models.js +1 -0
  85. package/dist/storage/implementation/v1/models.js.map +1 -1
  86. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  87. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  88. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  89. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  90. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  91. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
  93. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
  94. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  96. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  97. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
  99. package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
  100. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
  102. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
  103. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
  105. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
  106. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  107. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
  108. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
  109. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  110. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  111. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  112. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  113. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
  114. package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
  115. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  116. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  117. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  118. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  119. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
  120. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
  121. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  122. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  123. package/dist/storage/implementation/v3/bucket-format.js +79 -0
  124. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  125. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  126. package/dist/storage/implementation/v3/chunking.js +30 -0
  127. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  128. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  129. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  130. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  131. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  132. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  133. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  134. package/dist/storage/implementation/v3/models.d.ts +187 -23
  135. package/dist/storage/implementation/v3/models.js +11 -28
  136. package/dist/storage/implementation/v3/models.js.map +1 -1
  137. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
  138. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
  139. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  140. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
  141. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  142. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  143. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
  144. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
  145. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  146. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
  147. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
  148. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  149. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
  150. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
  151. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  152. package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
  153. package/dist/storage/implementation/v3/source-table-utils.js +206 -0
  154. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  155. package/dist/storage/storage-index.d.ts +5 -4
  156. package/dist/storage/storage-index.js +5 -4
  157. package/dist/storage/storage-index.js.map +1 -1
  158. package/dist/types/types.d.ts +71 -3
  159. package/dist/types/types.js +58 -2
  160. package/dist/types/types.js.map +1 -1
  161. package/dist/utils/test-utils.d.ts +1 -2
  162. package/dist/utils/test-utils.js +7 -1
  163. package/dist/utils/test-utils.js.map +1 -1
  164. package/dist/utils/util.d.ts +9 -2
  165. package/dist/utils/util.js +131 -5
  166. package/dist/utils/util.js.map +1 -1
  167. package/package.json +13 -19
  168. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  169. package/src/storage/MongoBucketStorage.ts +728 -242
  170. package/src/storage/implementation/MongoBucketBatch.ts +196 -91
  171. package/src/storage/implementation/MongoChecksums.ts +93 -153
  172. package/src/storage/implementation/MongoCompactor.ts +106 -631
  173. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  174. package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
  175. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
  176. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  177. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
  178. package/src/storage/implementation/MongoStorageProvider.ts +48 -4
  179. package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
  180. package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
  181. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
  182. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  183. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
  184. package/src/storage/implementation/common/PersistedBatch.ts +83 -15
  185. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  186. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  187. package/src/storage/implementation/db.ts +40 -22
  188. package/src/storage/implementation/models.ts +58 -14
  189. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
  190. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  191. package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
  192. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  193. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
  194. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  195. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  196. package/src/storage/implementation/v1/models.ts +12 -1
  197. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  198. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  199. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
  200. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  201. package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
  202. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
  203. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
  204. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
  205. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  206. package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
  207. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  208. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
  209. package/src/storage/implementation/v3/bucket-format.ts +99 -0
  210. package/src/storage/implementation/v3/chunking.ts +40 -0
  211. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  212. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  213. package/src/storage/implementation/v3/models.ts +228 -58
  214. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
  215. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
  216. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
  217. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
  218. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
  219. package/src/storage/implementation/v3/source-table-utils.ts +398 -0
  220. package/src/storage/storage-index.ts +14 -4
  221. package/src/types/types.ts +76 -2
  222. package/src/utils/test-utils.ts +8 -3
  223. package/src/utils/util.ts +188 -7
  224. package/test/src/__snapshots__/storage.test.ts.snap +4 -54
  225. package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
  226. package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
  227. package/test/src/compact-utils.test.ts +226 -0
  228. package/test/src/helpers/MemoryObjectStorage.ts +61 -0
  229. package/test/src/helpers/s3TestFactory.ts +69 -0
  230. package/test/src/object_storage_lifecycle.test.ts +59 -0
  231. package/test/src/object_storage_usage.test.ts +255 -0
  232. package/test/src/parameter_compacting_v1.test.ts +120 -0
  233. package/test/src/parameter_compacting_v3.test.ts +192 -0
  234. package/test/src/parameter_compaction_fence.test.ts +276 -0
  235. package/test/src/setup.ts +6 -1
  236. package/test/src/storage.test.ts +478 -13
  237. package/test/src/storage_compacting.test.ts +2516 -107
  238. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  239. package/test/src/storage_s3_checksums.test.ts +190 -0
  240. package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
  241. package/test/src/storage_s3_reading.test.ts +768 -0
  242. package/test/src/storage_s3_writing.test.ts +136 -0
  243. package/test/src/storage_sync.test.ts +1507 -76
  244. package/test/src/storeCurrentData.test.ts +227 -0
  245. package/test/src/util.ts +3 -2
  246. package/tsconfig.json +1 -4
  247. package/tsconfig.tsbuildinfo +1 -1
  248. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  249. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  250. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  251. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  252. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  253. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  254. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  255. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  256. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  257. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  258. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  259. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  260. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  261. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  262. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  263. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  264. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  265. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  266. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  267. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  268. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  269. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  270. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  271. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  272. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  273. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  274. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  275. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
package/src/utils/util.ts CHANGED
@@ -1,14 +1,56 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { mongo } from '@powersync/lib-service-mongodb';
3
- import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
3
+ import { Logger, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
4
4
  import { storage, utils } from '@powersync/service-core';
5
5
  import * as bson from 'bson';
6
6
  import * as crypto from 'crypto';
7
7
  import * as timers from 'node:timers/promises';
8
8
  import * as uuid from 'uuid';
9
9
  import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
10
+ import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
10
11
 
11
- export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
12
+ /**
13
+ * Default and max batch size for clearing.
14
+ */
15
+ const CLEAR_BATCH_SIZE = 10_000;
16
+
17
+ /**
18
+ * Minimum batch size for clearing when a batch takes too long.
19
+ */
20
+ const CLEAR_MIN_BATCH_SIZE = 100;
21
+
22
+ /**
23
+ * We target batch duration to stay between the min and max. If the druation is outside this range,
24
+ * we increase or decrease the batch size.
25
+ *
26
+ * MONGO_CLEAR_OPERATION_TIMEOUT_MS controls the absolute max of individual find and delete operations,
27
+ * but we try to avoid hitting that.
28
+ */
29
+ const CLEAR_BATCH_MIN_DURATION_MS = 500;
30
+ const CLEAR_BATCH_MAX_DURATION_MS = 2000;
31
+
32
+ type ClearCollectionFilter<T extends mongo.Document> = mongo.Filter<T> & {
33
+ _id: mongo.FilterOperators<mongo.InferIdType<T>>;
34
+ };
35
+
36
+ function throwIfClearAborted(signal?: AbortSignal): void {
37
+ if (signal?.aborted) {
38
+ throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
39
+ }
40
+ }
41
+
42
+ async function waitWithSignal(delayMs: number, signal: AbortSignal | undefined, abortMessage: string): Promise<void> {
43
+ try {
44
+ await timers.setTimeout(delayMs, undefined, { signal });
45
+ } catch (error) {
46
+ if (signal?.aborted) {
47
+ throw new ReplicationAbortedError(abortMessage, signal.reason);
48
+ }
49
+ throw error;
50
+ }
51
+ }
52
+
53
+ export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.FilterOperators<T> {
12
54
  let filter = {
13
55
  $gte: {
14
56
  ...prefix
@@ -26,9 +68,9 @@ export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.
26
68
  return filter;
27
69
  }
28
70
 
29
- export function generateSlotName(prefix: string, sync_rules_id: number) {
71
+ export function generateReplicationStreamName(prefix: string, replicationStreamId: number) {
30
72
  const slot_suffix = crypto.randomBytes(2).toString('hex');
31
- return `${prefix}${sync_rules_id}_${slot_suffix}`;
73
+ return `${prefix}${replicationStreamId}_${slot_suffix}`;
32
74
  }
33
75
 
34
76
  /**
@@ -79,7 +121,9 @@ export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry {
79
121
  object_type: row.table,
80
122
  object_id: row.row_id,
81
123
  checksum: Number(row.checksum),
82
- subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
124
+ // Storage V3 always persists this during replication. The fallback serves V1 data
125
+ // written before subkey persistence was introduced.
126
+ subkey: row.subkey ?? replicaIdToSubkey(row.source_table!, row.source_key!),
83
127
  data: row.data
84
128
  };
85
129
  } else {
@@ -141,6 +185,7 @@ export async function retryOnMongoMaxTimeMSExpired<T>(
141
185
  }
142
186
  ): Promise<T> {
143
187
  let retryCount = 0;
188
+ // Retry indefinitely on MaxTimeMSExpired errors with exponential backoff.
144
189
  while (true) {
145
190
  if (options.signal?.aborted) {
146
191
  throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
@@ -148,16 +193,152 @@ export async function retryOnMongoMaxTimeMSExpired<T>(
148
193
  try {
149
194
  return await operation();
150
195
  } catch (e) {
151
- if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
196
+ if (!lib_mongo.isMaxTimeMSExpiredError(e)) {
152
197
  throw e;
153
198
  }
154
199
  retryCount += 1;
155
200
  options.onRetry?.(retryCount);
156
- await timers.setTimeout(options.retryDelayMs);
201
+ await waitWithSignal(options.retryDelayMs, options.signal, options.abortMessage ?? 'Aborted MongoDB operation');
202
+ }
203
+ }
204
+ }
205
+
206
+ async function clearCollectionInBatches(
207
+ logger: Logger,
208
+ label: string,
209
+ findAndDeleteBatch: (batchSize: number) => Promise<number>,
210
+ signal?: AbortSignal,
211
+ throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
212
+ ): Promise<number> {
213
+ let batchSize = CLEAR_BATCH_SIZE;
214
+ let deletedCount = 0;
215
+ while (true) {
216
+ throwIfClearAborted(signal);
217
+ try {
218
+ const batchStartedAt = performance.now();
219
+ const foundBatchSize = await findAndDeleteBatch(batchSize);
220
+ const batchDurationMs = performance.now() - batchStartedAt;
221
+ deletedCount += foundBatchSize;
222
+ if (foundBatchSize > 0) {
223
+ logger.info(
224
+ `Cleared batch of ${label} (${foundBatchSize} documents) in ${Math.round(batchDurationMs)}ms, continuing...`
225
+ );
226
+ }
227
+ if (foundBatchSize < batchSize) {
228
+ return deletedCount;
229
+ }
230
+ if (batchDurationMs > CLEAR_BATCH_MAX_DURATION_MS) {
231
+ batchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
232
+ } else if (batchDurationMs < CLEAR_BATCH_MIN_DURATION_MS) {
233
+ batchSize = Math.min(CLEAR_BATCH_SIZE, batchSize * 2);
234
+ }
235
+ await waitWithSignal(batchDurationMs * throttleRate, signal, 'Aborted clearing data');
236
+ } catch (error) {
237
+ if (!lib_mongo.isTimeoutError(error) || batchSize == CLEAR_MIN_BATCH_SIZE) {
238
+ throw error;
239
+ }
240
+ const nextBatchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
241
+ logger.info(
242
+ `Clearing batch of ${label} timed out with batch size ${batchSize}, retrying find and delete with ${nextBatchSize}...`
243
+ );
244
+ batchSize = nextBatchSize;
245
+
246
+ // Pause for at least the equivalent of one query timeout before retrying, since the cause may be high database load.
247
+ const pauseDuration = Math.max(throttleRate, 1.0) * lib_mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS;
248
+ await waitWithSignal(pauseDuration, signal, 'Aborted clearing data');
157
249
  }
158
250
  }
159
251
  }
160
252
 
253
+ export async function clearCollectionInIdRanges<T extends mongo.Document>(
254
+ logger: Logger,
255
+ label: string,
256
+ collection: mongo.Collection<T>,
257
+ filter: ClearCollectionFilter<T>,
258
+ signal?: AbortSignal,
259
+ throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
260
+ ): Promise<number> {
261
+ return clearCollectionInBatches(
262
+ logger,
263
+ label,
264
+ async (batchSize) => {
265
+ const queryOptions: mongo.FindOptions<T> = {
266
+ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
267
+ projection: { _id: 1 }
268
+ };
269
+ const [batchEnd] = await collection
270
+ .find(filter, queryOptions)
271
+ .sort({ _id: 1 })
272
+ .skip(batchSize)
273
+ .limit(1)
274
+ .toArray();
275
+ throwIfClearAborted(signal);
276
+
277
+ if (batchEnd == null) {
278
+ // We're on the last batch
279
+ const result = await collection.deleteMany(filter, {
280
+ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
281
+ });
282
+ return result.deletedCount;
283
+ }
284
+
285
+ const idRange: mongo.FilterOperators<mongo.InferIdType<T>> = {
286
+ ...filter._id,
287
+ $lt: batchEnd._id
288
+ };
289
+ await collection.deleteMany(
290
+ {
291
+ ...filter,
292
+ _id: idRange
293
+ } as mongo.Filter<T>,
294
+ { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
295
+ );
296
+ return batchSize;
297
+ },
298
+ signal,
299
+ throttleRate
300
+ );
301
+ }
302
+
303
+ export async function clearCollectionInIdBatches<T extends mongo.Document>(
304
+ logger: Logger,
305
+ label: string,
306
+ collection: mongo.Collection<T>,
307
+ filter: mongo.Filter<T>,
308
+ signal?: AbortSignal,
309
+ throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
310
+ ): Promise<number> {
311
+ return clearCollectionInBatches(
312
+ logger,
313
+ label,
314
+ async (batchSize) => {
315
+ const documents = await collection
316
+ .find(filter, {
317
+ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
318
+ projection: { _id: 1 }
319
+ })
320
+ .limit(batchSize)
321
+ .toArray();
322
+ throwIfClearAborted(signal);
323
+
324
+ if (documents.length === 0) {
325
+ return 0;
326
+ }
327
+ await collection.deleteMany(
328
+ {
329
+ _id: {
330
+ $in: documents.map((document) => document._id)
331
+ }
332
+ } as mongo.Filter<T>,
333
+ { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
334
+ );
335
+ return documents.length;
336
+ },
337
+ signal,
338
+ throttleRate
339
+ );
340
+ }
341
+
161
342
  export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
162
343
  query: mongo.Filter<T>,
163
344
  collection: mongo.Collection<T>,
@@ -1,56 +1,6 @@
1
1
  // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2
2
 
3
- exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
4
- [
5
- {
6
- "checksum": 2871785649,
7
- "object_id": "test1",
8
- "op": "PUT",
9
- },
10
- {
11
- "checksum": 2872534815,
12
- "object_id": "test1",
13
- "op": "REMOVE",
14
- },
15
- {
16
- "checksum": 2871785649,
17
- "object_id": "test1",
18
- "op": "PUT",
19
- },
20
- {
21
- "checksum": 2872534815,
22
- "object_id": "test1",
23
- "op": "REMOVE",
24
- },
25
- ]
26
- `;
27
-
28
- exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
29
- [
30
- {
31
- "checksum": 2871785649,
32
- "object_id": "test1",
33
- "op": "PUT",
34
- },
35
- {
36
- "checksum": 2872534815,
37
- "object_id": "test1",
38
- "op": "REMOVE",
39
- },
40
- {
41
- "checksum": 2871785649,
42
- "object_id": "test1",
43
- "op": "PUT",
44
- },
45
- {
46
- "checksum": 2872534815,
47
- "object_id": "test1",
48
- "op": "REMOVE",
49
- },
50
- ]
51
- `;
52
-
53
- exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
3
+ exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
54
4
  [
55
5
  {
56
6
  "checksum": 2871785649,
@@ -75,7 +25,7 @@ exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3
75
25
  ]
76
26
  `;
77
27
 
78
- exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
28
+ exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
79
29
  [
80
30
  {
81
31
  "checksum": 2871785649,
@@ -100,7 +50,7 @@ exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (dele
100
50
  ]
101
51
  `;
102
52
 
103
- exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
53
+ exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
104
54
  [
105
55
  {
106
56
  "checksum": 2871785649,
@@ -125,7 +75,7 @@ exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (dele
125
75
  ]
126
76
  `;
127
77
 
128
- exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
78
+ exports[`Mongo Sync Bucket Storage - Data - v4 > (insert, delete, insert), (delete) 1`] = `
129
79
  [
130
80
  {
131
81
  "checksum": 2871785649,