@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +177 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +71 -18
  6. package/dist/storage/MongoBucketStorage.js +516 -202
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
  9. package/dist/storage/implementation/MongoBucketBatch.js +140 -66
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
  15. package/dist/storage/implementation/MongoCompactor.js +87 -481
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
  21. package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +41 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
  47. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  48. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  49. package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
  50. package/dist/storage/implementation/common/PersistedBatch.js +63 -10
  51. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +4 -1
  58. package/dist/storage/implementation/db.js +31 -16
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +55 -13
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  67. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  68. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
  69. package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
  70. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  72. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  73. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  74. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
  75. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
  76. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  77. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  78. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  79. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  80. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  81. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  82. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  83. package/dist/storage/implementation/v1/models.d.ts +10 -1
  84. package/dist/storage/implementation/v1/models.js +1 -0
  85. package/dist/storage/implementation/v1/models.js.map +1 -1
  86. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  87. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  88. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  89. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  90. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  91. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
  93. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
  94. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  96. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  97. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
  99. package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
  100. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
  102. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
  103. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
  105. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
  106. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  107. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
  108. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
  109. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  110. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  111. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  112. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  113. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
  114. package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
  115. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  116. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  117. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  118. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  119. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
  120. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
  121. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  122. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  123. package/dist/storage/implementation/v3/bucket-format.js +79 -0
  124. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  125. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  126. package/dist/storage/implementation/v3/chunking.js +30 -0
  127. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  128. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  129. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  130. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  131. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  132. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  133. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  134. package/dist/storage/implementation/v3/models.d.ts +187 -23
  135. package/dist/storage/implementation/v3/models.js +11 -28
  136. package/dist/storage/implementation/v3/models.js.map +1 -1
  137. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
  138. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
  139. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  140. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
  141. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  142. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  143. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
  144. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
  145. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  146. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
  147. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
  148. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  149. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
  150. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
  151. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  152. package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
  153. package/dist/storage/implementation/v3/source-table-utils.js +206 -0
  154. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  155. package/dist/storage/storage-index.d.ts +5 -4
  156. package/dist/storage/storage-index.js +5 -4
  157. package/dist/storage/storage-index.js.map +1 -1
  158. package/dist/types/types.d.ts +71 -3
  159. package/dist/types/types.js +58 -2
  160. package/dist/types/types.js.map +1 -1
  161. package/dist/utils/test-utils.d.ts +1 -2
  162. package/dist/utils/test-utils.js +7 -1
  163. package/dist/utils/test-utils.js.map +1 -1
  164. package/dist/utils/util.d.ts +9 -2
  165. package/dist/utils/util.js +131 -5
  166. package/dist/utils/util.js.map +1 -1
  167. package/package.json +13 -19
  168. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  169. package/src/storage/MongoBucketStorage.ts +728 -242
  170. package/src/storage/implementation/MongoBucketBatch.ts +196 -91
  171. package/src/storage/implementation/MongoChecksums.ts +93 -153
  172. package/src/storage/implementation/MongoCompactor.ts +106 -631
  173. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  174. package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
  175. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
  176. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  177. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
  178. package/src/storage/implementation/MongoStorageProvider.ts +48 -4
  179. package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
  180. package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
  181. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
  182. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  183. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
  184. package/src/storage/implementation/common/PersistedBatch.ts +83 -15
  185. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  186. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  187. package/src/storage/implementation/db.ts +40 -22
  188. package/src/storage/implementation/models.ts +58 -14
  189. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
  190. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  191. package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
  192. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  193. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
  194. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  195. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  196. package/src/storage/implementation/v1/models.ts +12 -1
  197. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  198. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  199. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
  200. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  201. package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
  202. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
  203. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
  204. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
  205. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  206. package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
  207. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  208. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
  209. package/src/storage/implementation/v3/bucket-format.ts +99 -0
  210. package/src/storage/implementation/v3/chunking.ts +40 -0
  211. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  212. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  213. package/src/storage/implementation/v3/models.ts +228 -58
  214. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
  215. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
  216. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
  217. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
  218. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
  219. package/src/storage/implementation/v3/source-table-utils.ts +398 -0
  220. package/src/storage/storage-index.ts +14 -4
  221. package/src/types/types.ts +76 -2
  222. package/src/utils/test-utils.ts +8 -3
  223. package/src/utils/util.ts +188 -7
  224. package/test/src/__snapshots__/storage.test.ts.snap +4 -54
  225. package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
  226. package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
  227. package/test/src/compact-utils.test.ts +226 -0
  228. package/test/src/helpers/MemoryObjectStorage.ts +61 -0
  229. package/test/src/helpers/s3TestFactory.ts +69 -0
  230. package/test/src/object_storage_lifecycle.test.ts +59 -0
  231. package/test/src/object_storage_usage.test.ts +255 -0
  232. package/test/src/parameter_compacting_v1.test.ts +120 -0
  233. package/test/src/parameter_compacting_v3.test.ts +192 -0
  234. package/test/src/parameter_compaction_fence.test.ts +276 -0
  235. package/test/src/setup.ts +6 -1
  236. package/test/src/storage.test.ts +478 -13
  237. package/test/src/storage_compacting.test.ts +2516 -107
  238. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  239. package/test/src/storage_s3_checksums.test.ts +190 -0
  240. package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
  241. package/test/src/storage_s3_reading.test.ts +768 -0
  242. package/test/src/storage_s3_writing.test.ts +136 -0
  243. package/test/src/storage_sync.test.ts +1507 -76
  244. package/test/src/storeCurrentData.test.ts +227 -0
  245. package/test/src/util.ts +3 -2
  246. package/tsconfig.json +1 -4
  247. package/tsconfig.tsbuildinfo +1 -1
  248. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  249. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  250. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  251. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  252. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  253. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  254. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  255. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  256. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  257. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  258. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  259. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  260. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  261. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  262. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  263. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  264. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  265. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  266. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  267. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  268. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  269. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  270. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  271. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  272. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  273. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  274. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  275. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,9 +1,55 @@
1
- import { mongo } from '@powersync/lib-service-mongodb';
2
- import { ReplicationAssertionError } from '@powersync/lib-services-framework';
1
+ import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
2
+ import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
3
+ import { addChecksums, isPartialChecksum, SyncRuleState, utils } from '@powersync/service-core';
3
4
  import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
4
5
  import { MongoCompactor } from '../MongoCompactor.js';
6
+ import { cacheKey } from '../OperationBatch.js';
5
7
  import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
8
+ const DEFAULT_MIN_BUCKET_CHANGES = 10;
9
+ const DEFAULT_MIN_CHANGE_RATIO = 0.1;
10
+ const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
6
11
  export class MongoCompactorV1 extends MongoCompactor {
12
+ options;
13
+ updates = [];
14
+ bucketStateUpdates = [];
15
+ minBucketChanges;
16
+ minChangeRatio;
17
+ maxOpId;
18
+ constructor(bucketStorage, db, options) {
19
+ super(bucketStorage, db, options);
20
+ this.options = options;
21
+ this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
22
+ this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
23
+ this.maxOpId = options.maxOpId ?? 0n;
24
+ }
25
+ /**
26
+ * Compact buckets by converting operations into MOVE and/or CLEAR operations.
27
+ *
28
+ * See /docs/storage/compacting-operations.md for details.
29
+ */
30
+ async compact() {
31
+ if (this.options?.incrementalOnly) {
32
+ // Not supported for V1
33
+ this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2');
34
+ return 0;
35
+ }
36
+ else if (this.storage.replicationStream.state != SyncRuleState.ACTIVE) {
37
+ this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`);
38
+ return 0;
39
+ }
40
+ await this.deleteOldCheckpointRequests();
41
+ if (this.buckets) {
42
+ for (const bucket of this.buckets) {
43
+ // We can make this more efficient later on by iterating through the buckets in a single query.
44
+ // That makes batching more tricky, so we leave for later.
45
+ await this.compactSingleBucketRetried(bucket);
46
+ }
47
+ }
48
+ else {
49
+ await this.compactDirtyBuckets();
50
+ }
51
+ return this.compactedBucketCount;
52
+ }
7
53
  async *dirtyBucketBatches(options) {
8
54
  if (options.minBucketChanges <= 0) {
9
55
  throw new ReplicationAssertionError('minBucketChanges must be >= 1');
@@ -11,7 +57,7 @@ export class MongoCompactorV1 extends MongoCompactor {
11
57
  // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes.
12
58
  // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is
13
59
  // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting.
14
- yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options, () => null);
60
+ yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options);
15
61
  }
16
62
  async dirtyBucketBatchForChecksums(options) {
17
63
  if (options.minBucketChanges <= 0) {
@@ -19,10 +65,186 @@ export class MongoCompactorV1 extends MongoCompactor {
19
65
  }
20
66
  // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
21
67
  // estimate_since_compact as it progresses.
22
- return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketStateV1, {
68
+ return this.dirtyBucketBatchForChecksumsForCollection({
23
69
  '_id.g': this.group_id,
24
70
  'estimate_since_compact.count': { $gte: options.minBucketChanges }
25
- }, () => null);
71
+ });
72
+ }
73
+ async *dirtyBucketBatchesForCollection(collection, lastId, maxId, options) {
74
+ // Paginate through the bucket state collection using cursor-based scanning.
75
+ while (true) {
76
+ // To avoid timeouts from too many buckets not meeting the minimum-change
77
+ // criteria, scan a fixed batch and only return matching buckets.
78
+ const [result] = await collection
79
+ .aggregate([
80
+ { $match: { _id: { $gt: lastId, $lt: maxId } } },
81
+ { $sort: { _id: 1 } },
82
+ // Scan a fixed number of docs each query so sparse matches don't block progress.
83
+ { $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE },
84
+ {
85
+ $facet: {
86
+ buckets: [
87
+ { $match: { 'estimate_since_compact.count': { $gte: options.minBucketChanges } } },
88
+ { $project: { _id: 1, estimate_since_compact: 1, compacted_state: 1 } }
89
+ ],
90
+ cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
91
+ }
92
+ }
93
+ ], { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS })
94
+ .toArray();
95
+ const cursor = result?.cursor?.[0];
96
+ if (cursor == null) {
97
+ break;
98
+ }
99
+ lastId = cursor._id;
100
+ const dirtyBuckets = (result?.buckets ?? []).map((bucketState) => {
101
+ // BigInt precision is not needed here since this is only an estimate.
102
+ const updatedCount = bucketState.estimate_since_compact?.count ?? 0;
103
+ const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount;
104
+ const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0);
105
+ const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes;
106
+ return {
107
+ bucket: bucketState._id.b,
108
+ definitionId: null,
109
+ estimatedCount: totalCount,
110
+ dirtyRatio: Math.max(totalCount > 0 ? updatedCount / totalCount : 0, totalBytes > 0 ? updatedBytes / totalBytes : 0)
111
+ };
112
+ });
113
+ yield dirtyBuckets.filter((bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio);
114
+ }
115
+ }
116
+ async dirtyBucketBatchForChecksumsForCollection(filter) {
117
+ const dirtyBuckets = await this.db.bucketStateV1
118
+ .find(filter, {
119
+ projection: { _id: 1, estimate_since_compact: 1, compacted_state: 1 },
120
+ sort: { 'estimate_since_compact.count': -1 },
121
+ limit: 200,
122
+ maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
123
+ })
124
+ .toArray();
125
+ return dirtyBuckets.map((bucket) => ({
126
+ bucket: bucket._id.b,
127
+ definitionId: null,
128
+ estimatedCount: Number(bucket.estimate_since_compact.count) + Number(bucket.compacted_state?.count ?? 0)
129
+ }));
130
+ }
131
+ async compactSingleBucketRetried(bucket, _definitionId = null) {
132
+ await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket));
133
+ this.compactedBucketCount++;
134
+ }
135
+ async compactDirtyBuckets() {
136
+ for await (const buckets of this.dirtyBucketBatches({
137
+ minBucketChanges: this.minBucketChanges,
138
+ minChangeRatio: this.minChangeRatio
139
+ })) {
140
+ this.signal?.throwIfAborted();
141
+ for (const { bucket, definitionId } of buckets) {
142
+ await this.compactSingleBucketRetried(bucket, definitionId);
143
+ }
144
+ }
145
+ }
146
+ /**
147
+ * Subset of compact, only populating checksums where relevant.
148
+ */
149
+ async populateChecksums(options) {
150
+ let count = 0;
151
+ // Paginate through dirty buckets in batches until no more buckets meet the criteria.
152
+ while (true) {
153
+ this.signal?.throwIfAborted();
154
+ const buckets = await this.dirtyBucketBatchForChecksums(options);
155
+ if (buckets.length == 0) {
156
+ break;
157
+ }
158
+ this.signal?.throwIfAborted();
159
+ const start = Date.now();
160
+ // Filter batch by estimated bucket size, to reduce possibility of timeouts.
161
+ const checkBuckets = [];
162
+ let totalCountEstimate = 0;
163
+ for (const bucket of buckets) {
164
+ checkBuckets.push(bucket);
165
+ totalCountEstimate += bucket.estimatedCount;
166
+ if (totalCountEstimate > 50_000) {
167
+ break;
168
+ }
169
+ }
170
+ this.logger.info(`Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`);
171
+ await this.updateChecksumsBatch(checkBuckets);
172
+ this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
173
+ count += checkBuckets.length;
174
+ }
175
+ return { buckets: count };
176
+ }
177
+ collectBucketStateUpdate(state, compactedOpId) {
178
+ if (state.opCount < 0) {
179
+ throw new ServiceAssertionError(`Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`);
180
+ }
181
+ return {
182
+ updateOne: {
183
+ filter: this.bucketStateFilter(state.bucket),
184
+ update: {
185
+ $set: {
186
+ compacted_state: {
187
+ op_id: compactedOpId,
188
+ count: state.opCount,
189
+ checksum: BigInt(state.checksum),
190
+ bytes: state.opBytes
191
+ },
192
+ estimate_since_compact: {
193
+ // There could have been a whole bunch of new operations added to the bucket while compacting,
194
+ // which we don't currently cater for. We could potentially query for that, but that adds overhead.
195
+ count: 0,
196
+ bytes: 0
197
+ }
198
+ }
199
+ },
200
+ // We generally expect this to have been created before.
201
+ // We don't create new ones here, to avoid issues with the unique index on bucket_updates.
202
+ upsert: false
203
+ }
204
+ };
205
+ }
206
+ updateBucketChecksums(state, compactedOpId) {
207
+ this.bucketStateUpdates.push(this.collectBucketStateUpdate(state, compactedOpId));
208
+ }
209
+ async flushBucketStateUpdates() {
210
+ if (this.bucketStateUpdates.length > 0) {
211
+ this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
212
+ await this.writeBucketStateUpdates();
213
+ this.bucketStateUpdates = [];
214
+ }
215
+ }
216
+ async updateChecksumsBatch(buckets) {
217
+ const checksums = await this.computeChecksumsForBuckets(buckets);
218
+ const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
219
+ for (const bucketChecksum of checksums.values()) {
220
+ if (isPartialChecksum(bucketChecksum)) {
221
+ // Should never happen since we don't specify `start`.
222
+ throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`);
223
+ }
224
+ this.bucketStateUpdates.push({
225
+ updateOne: {
226
+ filter: this.bucketStateFilter(bucketChecksum.bucket),
227
+ update: {
228
+ $set: {
229
+ compacted_state: {
230
+ op_id: this.maxOpId,
231
+ count: bucketChecksum.count,
232
+ checksum: BigInt(bucketChecksum.checksum),
233
+ bytes: null
234
+ },
235
+ estimate_since_compact: {
236
+ count: 0,
237
+ bytes: 0
238
+ }
239
+ }
240
+ },
241
+ // We don't create new ones here - it gets tricky to get the last_op right with the unique index on
242
+ // bucket_updates.
243
+ upsert: false
244
+ }
245
+ });
246
+ }
247
+ await this.flushBucketStateUpdates();
26
248
  }
27
249
  async writeBucketStateUpdates() {
28
250
  await this.db.bucketStateV1.bulkWrite(this.bucketStateUpdates, { ordered: false });
@@ -31,9 +253,9 @@ export class MongoCompactorV1 extends MongoCompactor {
31
253
  return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
32
254
  bucket,
33
255
  end: this.maxOpId
34
- })));
256
+ })), { readOptions: { readConcern: 'snapshot' } });
35
257
  }
36
- bucketStateFilter(bucket, _definitionId) {
258
+ bucketStateFilter(bucket) {
37
259
  return {
38
260
  _id: {
39
261
  g: this.group_id,
@@ -41,12 +263,260 @@ export class MongoCompactorV1 extends MongoCompactor {
41
263
  }
42
264
  };
43
265
  }
44
- async getBucketDataContext(bucket, _definitionId) {
266
+ getBucketDataContext(bucket) {
45
267
  return new SingleBucketStoreV1(this.db, {
46
268
  replicationStreamId: this.group_id,
47
269
  definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
48
270
  bucket
49
271
  });
50
272
  }
273
+ async compactSingleBucket(bucket) {
274
+ // Do not carry queued writes from a failed attempt into the rescan.
275
+ this.updates = [];
276
+ this.bucketStateUpdates = [];
277
+ const idLimitBytes = this.idLimitBytes;
278
+ const bucketContext = this.getBucketDataContext(bucket);
279
+ const currentState = {
280
+ bucket,
281
+ definitionId: bucketContext.key.definitionId,
282
+ seen: new Map(),
283
+ trackingSize: 0,
284
+ lastNotPut: null,
285
+ opsSincePut: 0,
286
+ checksum: 0,
287
+ opCount: 0,
288
+ opBytes: 0
289
+ };
290
+ // Constant lower bound.
291
+ const lowerBound = bucketContext.minId;
292
+ // Upper bound is adjusted for each batch.
293
+ let upperBound = bucketContext.maxId;
294
+ // Paginate through bucket data in batches to avoid cursor timeouts.
295
+ while (true) {
296
+ this.signal?.throwIfAborted();
297
+ // Query one batch at a time, to avoid cursor timeouts.
298
+ const pipeline = [
299
+ {
300
+ $match: {
301
+ _id: {
302
+ $gte: lowerBound,
303
+ $lt: upperBound
304
+ }
305
+ }
306
+ },
307
+ { $sort: { _id: -1 } },
308
+ { $limit: this.moveBatchQueryLimit },
309
+ {
310
+ $project: {
311
+ _id: 1,
312
+ op: 1,
313
+ table: 1,
314
+ row_id: 1,
315
+ source_table: 1,
316
+ source_key: 1,
317
+ checksum: 1,
318
+ size: { $bsonSize: '$$ROOT' }
319
+ }
320
+ }
321
+ ];
322
+ const cursor = bucketContext.collection.aggregate(pipeline, {
323
+ // batchSize is 1 more than limit to auto-close the cursor.
324
+ // See https://github.com/mongodb/node-mongodb-native/pull/4580
325
+ batchSize: this.moveBatchQueryLimit + 1
326
+ });
327
+ // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
328
+ // Instead, we load up to the limit.
329
+ const rawBatch = await cursor.toArray();
330
+ const batch = rawBatch.map((document) => {
331
+ const { size, ...rest } = document;
332
+ return {
333
+ doc: bucketContext.fromPersistedDocument(rest),
334
+ size
335
+ };
336
+ });
337
+ if (batch.length == 0) {
338
+ // We've reached the end.
339
+ break;
340
+ }
341
+ // Reuse the exact collection _id value from Mongo for the next bound.
342
+ upperBound = rawBatch[rawBatch.length - 1]._id;
343
+ for (const { doc, size } of batch) {
344
+ if (doc.o > this.maxOpId) {
345
+ continue;
346
+ }
347
+ currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
348
+ currentState.opCount += 1;
349
+ let isPersistentPut = doc.op == 'PUT';
350
+ currentState.opBytes += Number(size);
351
+ if (doc.op == 'REMOVE' || doc.op == 'PUT') {
352
+ const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table, doc.source_key)}`;
353
+ const targetOp = currentState.seen.get(key);
354
+ if (targetOp) {
355
+ // Will convert to MOVE, so don't count as PUT.
356
+ isPersistentPut = false;
357
+ this.updates.push({
358
+ updateOne: {
359
+ filter: { _id: bucketContext.docId(doc.o) },
360
+ update: {
361
+ $set: {
362
+ op: 'MOVE',
363
+ target_op: targetOp
364
+ },
365
+ $unset: {
366
+ source_table: 1,
367
+ source_key: 1,
368
+ subkey: 1,
369
+ table: 1,
370
+ row_id: 1,
371
+ data: 1
372
+ }
373
+ }
374
+ }
375
+ });
376
+ // TODO: better estimate for this.
377
+ currentState.opBytes += 200 - Number(size);
378
+ }
379
+ else if (currentState.trackingSize < idLimitBytes) {
380
+ // flatstr reduces the memory usage by flattening the string.
381
+ currentState.seen.set(utils.flatstr(key), doc.o);
382
+ // length + 16 for the string
383
+ // 24 for the bigint
384
+ // 50 for map overhead
385
+ // 50 for additional overhead
386
+ currentState.trackingSize += key.length + 140;
387
+ }
388
+ }
389
+ if (isPersistentPut) {
390
+ currentState.lastNotPut = null;
391
+ currentState.opsSincePut = 0;
392
+ }
393
+ else if (doc.op != 'CLEAR') {
394
+ if (currentState.lastNotPut == null) {
395
+ currentState.lastNotPut = doc.o;
396
+ }
397
+ currentState.opsSincePut += 1;
398
+ }
399
+ if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
400
+ await this.flush(bucketContext);
401
+ }
402
+ }
403
+ this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
404
+ }
405
+ // Free memory before clearing the bucket.
406
+ currentState.seen.clear();
407
+ if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
408
+ this.logger.info(`Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
409
+ // Need flush() before clear().
410
+ await this.flush(bucketContext);
411
+ await this.clearBucket(currentState, bucketContext);
412
+ }
413
+ // Do this after clearBucket so we have accurate counts.
414
+ this.updateBucketChecksums(currentState, this.maxOpId);
415
+ // Need another flush after updateBucketChecksums().
416
+ await this.flush(bucketContext);
417
+ }
418
+ async flush(bucketContext) {
419
+ if (this.updates.length > 0) {
420
+ this.logger.info(`Compacting ${this.updates.length} ops`);
421
+ await bucketContext.collection.bulkWrite(this.updates, {
422
+ // Order is not important. Since checksums are not affected, these operations can happen in any order,
423
+ // and it's fine if the operations are partially applied. Each individual operation is atomic.
424
+ ordered: false
425
+ });
426
+ this.updates = [];
427
+ }
428
+ await this.flushBucketStateUpdates();
429
+ }
430
+ /**
431
+ * Perform a CLEAR compact for a bucket.
432
+ *
433
+ * @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
434
+ */
435
+ async clearBucket(currentState, bucketContext) {
436
+ const clearOp = currentState.lastNotPut;
437
+ const opFilter = {
438
+ _id: {
439
+ $gte: bucketContext.minId,
440
+ $lte: bucketContext.docId(clearOp)
441
+ }
442
+ };
443
+ const session = this.db.client.startSession();
444
+ try {
445
+ let done = false;
446
+ while (!done) {
447
+ this.signal?.throwIfAborted();
448
+ let opCountDiff = 0;
449
+ // Do the CLEAR operation in batches, with each batch a separate transaction.
450
+ // The state after each batch is fully consistent.
451
+ // We need a transaction per batch to make sure checksums stay consistent.
452
+ await session.withTransaction(async () => {
453
+ const query = bucketContext.collection.find(opFilter, {
454
+ session,
455
+ sort: { _id: 1 },
456
+ projection: {
457
+ _id: 1,
458
+ op: 1,
459
+ checksum: 1,
460
+ target_op: 1
461
+ },
462
+ limit: this.clearBatchLimit
463
+ });
464
+ let checksum = 0;
465
+ let lastOp = null;
466
+ let targetOp = null;
467
+ let gotAnOp = false;
468
+ let numberOfOpsToClear = 0;
469
+ for await (const rawOp of query.stream()) {
470
+ const op = bucketContext.fromPartialPersistedDocument(rawOp);
471
+ if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
472
+ checksum = utils.addChecksums(checksum, Number(op.checksum));
473
+ lastOp = op;
474
+ numberOfOpsToClear += 1;
475
+ if (op.op != 'CLEAR') {
476
+ gotAnOp = true;
477
+ }
478
+ if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
479
+ targetOp = op.target_op;
480
+ }
481
+ }
482
+ else {
483
+ throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
484
+ }
485
+ }
486
+ if (!gotAnOp) {
487
+ done = true;
488
+ return;
489
+ }
490
+ this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
491
+ await bucketContext.collection.deleteMany({
492
+ _id: {
493
+ $gte: bucketContext.minId,
494
+ $lte: bucketContext.docId(lastOp.o)
495
+ }
496
+ }, { session });
497
+ const op = bucketContext.toPersistedDocument({
498
+ o: lastOp.o,
499
+ op: 'CLEAR',
500
+ checksum: BigInt(checksum),
501
+ data: null,
502
+ target_op: targetOp
503
+ });
504
+ await bucketContext.collection.insertOne(op, { session });
505
+ opCountDiff = -numberOfOpsToClear + 1;
506
+ }, {
507
+ writeConcern: { w: 'majority' },
508
+ readConcern: { level: 'snapshot' }
509
+ });
510
+ // Update outside the transaction, since the transaction can be retried multiple times.
511
+ currentState.opCount += opCountDiff;
512
+ }
513
+ }
514
+ finally {
515
+ await session.endSession();
516
+ }
517
+ }
518
+ formatBucketDataKey(doc) {
519
+ return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
520
+ }
51
521
  }
52
522
  //# sourceMappingURL=MongoCompactorV1.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAI9E,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,aAAwC;QAExC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACrG,OAAO,EACL,YAAY,EAGZ,iBAAiB,EAEjB,aAAa,EACb,KAAK,EACN,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAuB,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAK/D,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AACrC,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAqB3C,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAcxC;IATF,OAAO,GAAwD,EAAE,CAAC;IAClE,kBAAkB,GAA2D,EAAE,CAAC;IACvE,gBAAgB,CAAS;IACzB,cAAc,CAAS;IACvB,OAAO,CAAS;IAEjC,YACE,aAAuC,EACvC,EAA6B,EACrB,OAA4B;QAEpC,KAAK,CAAC,aAAa,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QAF1B,YAAO,GAAP,OAAO,CAAqB;QAGpC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,0BAA0B,CAAC;QAC/E,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACzE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,OAAO;QACpB,IAAI,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC;YAClC,uBAAuB;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;YACrF,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,SAAS,CAAC,CAAC;YAChH,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEzC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClC,+FAA+F;gBAC/F,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,CACR,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CAAC;YACpD,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,+BAA+B,CAC5C,UAA0C,EAC1C,MAA2B,EAC3B,KAA0B,EAC1B,OAA6D;QAE7D,4EAA4E;QAC5E,OAAO,IAAI,EAAE,CAAC;YACZ,yEAAyE;YACzE,iEAAiE;YACjE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU;iBAC9B,SAAS,CACR;gBACE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE;gBAChD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,iFAAiF;gBACjF,EAAE,MAAM,EAAE,4BAA4B,EAAE;gBACxC;oBACE,MAAM,EAAE;wBACN,OAAO,EAAE;4BACP,EAAE,MAAM,EAAE,EAAE,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE,EAAE,EAAE;4BAClF,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,EAAE;yBACxE;wBACD,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;qBAC1E;iBACF;aACF,EACD,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAC1C;iBACA,OAAO,EAAE,CAAC;YAEb,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC;YAEpB,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;gBAC/D,sEAAsE;gBACtE,MAAM,YAAY,GAAG,WAAW,CAAC,sBAAsB,EAAE,KAAK,IAAI,CAAC,CAAC;gBACpE,MAAM,UAAU,GAAG,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC;gBAC5E,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,sBAAsB,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;gBAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC;gBAClF,OAAO;oBACL,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;oBACzB,YAAY,EAAE,IAAI;oBAClB,cAAc,EAAE,UAAU;oBAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,CAClB,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,EAC9C,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAC/C;iBACF,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,MAAM,CACvB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,UAAU,IAAI,OAAO,CAAC,cAAc,CAC7G,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yCAAyC,CACrD,MAA2C;QAE3C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa;aAC7C,IAAI,CAAC,MAAM,EAAE;YACZ,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE;YACrE,IAAI,EAAE,EAAE,8BAA8B,EAAE,CAAC,CAAC,EAAE;YAC5C,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,0BAA0B;SACtC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,YAAY,EAAE,IAAI;YAClB,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,sBAAuB,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC;SAC1G,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,MAAc,EAAE,gBAA2C,IAAI;QACtG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,IAAI,CAAC,kBAAkB,CAAC;YAClD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC,CAAC,EAAE,CAAC;YACH,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,KAAK,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,OAAO,EAAE,CAAC;gBAC/C,MAAM,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAAqC;QAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,qFAAqF;QACrF,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;YACjE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,4EAA4E;YAC5E,MAAM,YAAY,GAAmB,EAAE,CAAC;YACxC,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1B,kBAAkB,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC5C,IAAI,kBAAkB,GAAG,MAAM,EAAE,CAAC;oBAChC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sCAAsC,OAAO,CAAC,MAAM,gCAAgC,kBAAkB,EAAE,CACzG,CAAC;YACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,YAAY,CAAC,MAAM,eAAe,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;YAC7G,KAAK,IAAI,YAAY,CAAC,MAAM,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAEO,wBAAwB,CAC9B,KAAyB,EACzB,aAAqB;QAErB,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,OAAO;YACL,SAAS,EAAE;gBACT,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC;gBAC5C,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,aAAa;4BACpB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,8FAA8F;4BAC9F,mGAAmG;4BACnG,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;iBACoD;gBACvD,wDAAwD;gBACxD,0FAA0F;gBAC1F,MAAM,EAAE,KAAK;aACd;SACF,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAAC,KAAyB,EAAE,aAAqB;QAC5E,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,uBAAuB;QACnC,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YAC7E,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACrC,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAuD;QACxF,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEpG,KAAK,MAAM,cAAc,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,sDAAsD;gBACtD,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC3B,SAAS,EAAE;oBACT,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,MAAM,CAAC;oBACrD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,eAAe,EAAE;gCACf,KAAK,EAAE,IAAI,CAAC,OAAO;gCACnB,KAAK,EAAE,cAAc,CAAC,KAAK;gCAC3B,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;gCACzC,KAAK,EAAE,IAAI;6BACZ;4BACD,sBAAsB,EAAE;gCACtB,KAAK,EAAE,CAAC;gCACR,KAAK,EAAE,CAAC;6BACT;yBACF;qBACoD;oBACvD,mGAAmG;oBACnG,kBAAkB;oBAClB,MAAM,EAAE,KAAK;iBACd;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,EACH,EAAE,WAAW,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,CAC7C,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,MAAc;QACtC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,MAAc;QACzC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAChD,oEAAoE;QACpE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAE7B,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,YAAY,GAAuB;YACvC,MAAM;YACN,YAAY,EAAE,aAAa,CAAC,GAAG,CAAC,YAAY;YAC5C,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,CAAC;QAEF,wBAAwB;QACxB,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QACvC,0CAA0C;QAC1C,IAAI,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QAErC,oEAAoE;QACpE,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,uDAAuD;YACvD,MAAM,QAAQ,GAAG;gBACf;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,SAAS,CAAmD,QAAQ,EAAE;gBAC5G,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,mBAAmB,GAAG,CAAC;aACxC,CAAC,CAAC;YACH,+GAA+G;YAC/G,oCAAoC;YACpC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACtC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;gBACnC,OAAO;oBACL,GAAG,EAAE,aAAa,CAAC,qBAAqB,CAAC,IAAI,CAAC;oBAC9C,IAAI;iBACL,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,yBAAyB;gBACzB,MAAM;YACR,CAAC;YAED,sEAAsE;YACtE,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAE/C,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,+CAA+C;wBAC/C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gCAC3C,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,MAAM,EAAE,CAAC;wCACT,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACiD;6BACrD;yBACF,CAAC,CAAC;wBAEH,kCAAkC;wBAClC,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC7C,CAAC;yBAAM,IAAI,YAAY,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC;wBACpD,6DAA6D;wBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;wBACjD,6BAA6B;wBAC7B,oBAAoB;wBACpB,sBAAsB;wBACtB,6BAA6B;wBAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;oBAChD,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC;oBAClC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,MAAM,oBAAoB,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,0CAA0C;QAC1C,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sBAAsB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CAC5H,CAAC;YACF,+BAA+B;YAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACtD,CAAC;QAED,wDAAwD;QACxD,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,oDAAoD;QACpD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,aAAkC;QACpD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YAC1D,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBACrD,sGAAsG;gBACtG,8FAA8F;gBAC9F,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC,EAAE,aAAkC;QAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE,aAAa,CAAC,KAAK;gBACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;aACnC;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;gBAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CACzC,QAAQ,EACR;wBACE,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CACF,CAAC;oBACF,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,MAAM,GAA6D,IAAI,CAAC;oBAC5E,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACzC,MAAM,EAAE,GAAG,aAAa,CAAC,4BAA4B,CAAC,KAAK,CAAC,CAAC;wBAE7D,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,MAAM,GAAG,EAAE,CAAC;4BACZ,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;gCAC1E,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;4BAC1B,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1G,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;oBACjF,MAAM,aAAa,CAAC,UAAU,CAAC,UAAU,CACvC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE,aAAa,CAAC,KAAK;4BACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC;yBACrC;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;wBAC3C,CAAC,EAAE,MAAO,CAAC,CAAC;wBACZ,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,CAAC,CAAC;oBACH,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;oBAE1D,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,uFAAuF;gBACvF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,GAA2C;QACrE,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,mBAAmB,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;IACjF,CAAC;CACF"}
@@ -1,9 +1,23 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { bson, InternalOpId } from '@powersync/service-core';
2
3
  import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
3
4
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
4
5
  export declare class MongoParameterCompactorV1 extends MongoParameterCompactor {
5
6
  protected readonly db: VersionedPowerSyncMongoV1;
6
7
  protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
7
- protected collectionFilter(): mongo.Document;
8
- protected deleteFilter(doc: mongo.Document): mongo.Document;
8
+ protected shouldCompactDocument(doc: {
9
+ key: mongo.Document;
10
+ }): boolean;
11
+ /**
12
+ * Uses the legacy `{ 'key.g': 1, lookup: 1, _id: 1 }` index to narrow the stream, lookup and
13
+ * operation-id range. `key` is not part of that index at all, so it is a residual predicate applied
14
+ * to every document the range scan returns.
15
+ *
16
+ * That scan may therefore have to filter through many keys for the same lookup, but the cost is
17
+ * amortized: a single scan covers up to 1000 keys, and identities seen again in a later batch skip
18
+ * the scan entirely - they are deleted by `_id`.
19
+ *
20
+ * The V3 storage format uses an index more suitable for this.
21
+ */
22
+ protected leadingHistoryDeleteFilter(lookup: bson.Binary, keys: mongo.Document[], before: InternalOpId): mongo.Document;
9
23
  }
@@ -3,17 +3,28 @@ export class MongoParameterCompactorV1 extends MongoParameterCompactor {
3
3
  async getCollections() {
4
4
  return [this.db.parameterIndexV1];
5
5
  }
6
- collectionFilter() {
7
- return {
8
- 'key.g': this.group_id
9
- };
6
+ // The shared V1 collection is scanned using only its default `_id` index. Filter the stream in
7
+ // code so compaction does not require an index on `key.g`.
8
+ shouldCompactDocument(doc) {
9
+ return doc.key.g === this.replicationStreamId;
10
10
  }
11
- deleteFilter(doc) {
11
+ /**
12
+ * Uses the legacy `{ 'key.g': 1, lookup: 1, _id: 1 }` index to narrow the stream, lookup and
13
+ * operation-id range. `key` is not part of that index at all, so it is a residual predicate applied
14
+ * to every document the range scan returns.
15
+ *
16
+ * That scan may therefore have to filter through many keys for the same lookup, but the cost is
17
+ * amortized: a single scan covers up to 1000 keys, and identities seen again in a later batch skip
18
+ * the scan entirely - they are deleted by `_id`.
19
+ *
20
+ * The V3 storage format uses an index more suitable for this.
21
+ */
22
+ leadingHistoryDeleteFilter(lookup, keys, before) {
12
23
  return {
13
- 'key.g': doc.key.g,
14
- lookup: doc.lookup,
15
- _id: { $lte: doc._id },
16
- key: doc.key
24
+ 'key.g': this.replicationStreamId,
25
+ lookup,
26
+ key: { $in: keys },
27
+ _id: { $lt: before }
17
28
  };
18
29
  }
19
30
  }
@@ -1 +1 @@
1
- {"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,gBAA+D,CAAC,CAAC;IACnF,CAAC;IAES,gBAAgB;QACxB,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC;IACJ,CAAC;IAES,YAAY,CAAC,GAAmB;QACxC,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAW;YAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG,EAAE,GAAG,CAAC,GAAG;SACb,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,gBAA+D,CAAC,CAAC;IACnF,CAAC;IAED,+FAA+F;IAC/F,2DAA2D;IACxC,qBAAqB,CAAC,GAA4B;QACnE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,mBAAmB,CAAC;IAChD,CAAC;IAED;;;;;;;;;;OAUG;IACO,0BAA0B,CAClC,MAAmB,EACnB,IAAsB,EACtB,MAAoB;QAEpB,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,mBAAmB;YACjC,MAAM;YACN,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YAClB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE;SACrB,CAAC;IACJ,CAAC;CACF"}