@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,20 +1,25 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
3
- import { storage } from '@powersync/service-core';
3
+ import { addChecksums, storage, utils } from '@powersync/service-core';
4
4
  import { BucketDefinitionId } from '@powersync/service-sync-rules';
5
- import { SingleBucketStore } from '../common/SingleBucketStore.js';
5
+ import { BucketDataDoc } from '../common/BucketDataDoc.js';
6
6
  import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
7
- import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
8
- import { BucketStateDocumentV1 } from './models.js';
7
+ import { CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
8
+ import { cacheKey } from '../OperationBatch.js';
9
+ import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js';
9
10
  import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
10
11
  import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
11
12
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
12
13
 
14
+ type CompactClearProperties = 'op' | 'checksum' | 'target_op';
15
+
13
16
  export class MongoCompactorV1 extends MongoCompactor {
14
17
  // Override types to the more specific ones
15
18
  declare protected readonly db: VersionedPowerSyncMongoV1;
16
19
  declare protected readonly storage: MongoSyncBucketStorageV1;
17
20
 
21
+ private updates: mongo.AnyBulkWriteOperation<BucketDataDocumentV1>[] = [];
22
+
18
23
  public async *dirtyBucketBatches(options: {
19
24
  minBucketChanges: number;
20
25
  minChangeRatio: number;
@@ -64,7 +69,8 @@ export class MongoCompactorV1 extends MongoCompactor {
64
69
  buckets.map(({ bucket }) => ({
65
70
  bucket,
66
71
  end: this.maxOpId
67
- }))
72
+ })),
73
+ { readOptions: { readConcern: 'snapshot' } }
68
74
  );
69
75
  }
70
76
 
@@ -80,14 +86,293 @@ export class MongoCompactorV1 extends MongoCompactor {
80
86
  };
81
87
  }
82
88
 
83
- protected async getBucketDataContext(
84
- bucket: string,
85
- _definitionId: BucketDefinitionId | null
86
- ): Promise<SingleBucketStore | null> {
89
+ private getBucketDataContext(bucket: string): SingleBucketStoreV1 {
87
90
  return new SingleBucketStoreV1(this.db, {
88
91
  replicationStreamId: this.group_id,
89
92
  definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
90
93
  bucket
91
94
  });
92
95
  }
96
+
97
+ protected async compactSingleBucket(bucket: string) {
98
+ // Do not carry queued writes from a failed attempt into the rescan
99
+ this.updates = [];
100
+
101
+ const idLimitBytes = this.idLimitBytes;
102
+ const bucketContext = this.getBucketDataContext(bucket);
103
+ const currentState: CurrentBucketState = {
104
+ bucket,
105
+ definitionId: bucketContext.key.definitionId,
106
+ seen: new Map(),
107
+ trackingSize: 0,
108
+ lastNotPut: null,
109
+ opsSincePut: 0,
110
+ checksum: 0,
111
+ opCount: 0,
112
+ opBytes: 0
113
+ };
114
+
115
+ // Constant lower bound.
116
+ const lowerBound = bucketContext.minId;
117
+ // Upper bound is adjusted for each batch.
118
+ let upperBound = bucketContext.maxId;
119
+
120
+ // Paginate through bucket data in batches to avoid cursor timeouts.
121
+ while (true) {
122
+ this.signal?.throwIfAborted();
123
+
124
+ // Query one batch at a time, to avoid cursor timeouts.
125
+ const pipeline = [
126
+ {
127
+ $match: {
128
+ _id: {
129
+ $gte: lowerBound,
130
+ $lt: upperBound
131
+ }
132
+ }
133
+ },
134
+ { $sort: { _id: -1 } },
135
+ { $limit: this.moveBatchQueryLimit },
136
+ {
137
+ $project: {
138
+ _id: 1,
139
+ op: 1,
140
+ table: 1,
141
+ row_id: 1,
142
+ source_table: 1,
143
+ source_key: 1,
144
+ checksum: 1,
145
+ size: { $bsonSize: '$$ROOT' }
146
+ }
147
+ }
148
+ ];
149
+
150
+ const cursor = bucketContext.collection.aggregate<BucketDataDocumentV1 & { size: number | bigint }>(pipeline, {
151
+ // batchSize is 1 more than limit to auto-close the cursor.
152
+ // See https://github.com/mongodb/node-mongodb-native/pull/4580
153
+ batchSize: this.moveBatchQueryLimit + 1
154
+ });
155
+ // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
156
+ // Instead, we load up to the limit.
157
+ const rawBatch = await cursor.toArray();
158
+ const batch = rawBatch.map((document) => {
159
+ const { size, ...rest } = document;
160
+ return {
161
+ doc: bucketContext.fromPersistedDocument(rest),
162
+ size
163
+ };
164
+ });
165
+
166
+ if (batch.length == 0) {
167
+ // We've reached the end.
168
+ break;
169
+ }
170
+
171
+ // Reuse the exact collection _id value from Mongo for the next bound.
172
+ upperBound = rawBatch[rawBatch.length - 1]._id;
173
+
174
+ for (const { doc, size } of batch) {
175
+ if (doc.o > this.maxOpId) {
176
+ continue;
177
+ }
178
+
179
+ currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
180
+ currentState.opCount += 1;
181
+
182
+ let isPersistentPut = doc.op == 'PUT';
183
+
184
+ currentState.opBytes += Number(size);
185
+ if (doc.op == 'REMOVE' || doc.op == 'PUT') {
186
+ const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`;
187
+ const targetOp = currentState.seen.get(key);
188
+ if (targetOp) {
189
+ // Will convert to MOVE, so don't count as PUT.
190
+ isPersistentPut = false;
191
+
192
+ this.updates.push({
193
+ updateOne: {
194
+ filter: { _id: bucketContext.docId(doc.o) },
195
+ update: {
196
+ $set: {
197
+ op: 'MOVE',
198
+ target_op: targetOp
199
+ },
200
+ $unset: {
201
+ source_table: 1,
202
+ source_key: 1,
203
+ table: 1,
204
+ row_id: 1,
205
+ data: 1
206
+ }
207
+ } satisfies mongo.UpdateFilter<BucketDataDocumentV1>
208
+ }
209
+ });
210
+
211
+ // TODO: better estimate for this.
212
+ currentState.opBytes += 200 - Number(size);
213
+ } else if (currentState.trackingSize < idLimitBytes) {
214
+ // flatstr reduces the memory usage by flattening the string.
215
+ currentState.seen.set(utils.flatstr(key), doc.o);
216
+ // length + 16 for the string
217
+ // 24 for the bigint
218
+ // 50 for map overhead
219
+ // 50 for additional overhead
220
+ currentState.trackingSize += key.length + 140;
221
+ }
222
+ }
223
+
224
+ if (isPersistentPut) {
225
+ currentState.lastNotPut = null;
226
+ currentState.opsSincePut = 0;
227
+ } else if (doc.op != 'CLEAR') {
228
+ if (currentState.lastNotPut == null) {
229
+ currentState.lastNotPut = doc.o;
230
+ }
231
+ currentState.opsSincePut += 1;
232
+ }
233
+
234
+ if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
235
+ await this.flush(bucketContext);
236
+ }
237
+ }
238
+
239
+ this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
240
+ }
241
+
242
+ // Free memory before clearing the bucket.
243
+ currentState.seen.clear();
244
+ if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
245
+ this.logger.info(
246
+ `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
247
+ );
248
+ // Need flush() before clear().
249
+ await this.flush(bucketContext);
250
+ await this.clearBucket(currentState, bucketContext);
251
+ }
252
+
253
+ // Do this after clearBucket so we have accurate counts.
254
+ this.updateBucketChecksums(currentState, this.maxOpId);
255
+ // Need another flush after updateBucketChecksums().
256
+ await this.flush(bucketContext);
257
+ }
258
+
259
+ private async flush(bucketContext: SingleBucketStoreV1) {
260
+ if (this.updates.length > 0) {
261
+ this.logger.info(`Compacting ${this.updates.length} ops`);
262
+ await bucketContext.collection.bulkWrite(this.updates, {
263
+ // Order is not important. Since checksums are not affected, these operations can happen in any order,
264
+ // and it's fine if the operations are partially applied. Each individual operation is atomic.
265
+ ordered: false
266
+ });
267
+ this.updates = [];
268
+ }
269
+
270
+ await this.flushBucketStateUpdates();
271
+ }
272
+
273
+ /**
274
+ * Perform a CLEAR compact for a bucket.
275
+ *
276
+ * @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
277
+ */
278
+ private async clearBucket(currentState: CurrentBucketState, bucketContext: SingleBucketStoreV1) {
279
+ const clearOp = currentState.lastNotPut!;
280
+
281
+ const opFilter = {
282
+ _id: {
283
+ $gte: bucketContext.minId,
284
+ $lte: bucketContext.docId(clearOp)
285
+ }
286
+ };
287
+
288
+ const session = this.db.client.startSession();
289
+ try {
290
+ let done = false;
291
+ while (!done) {
292
+ this.signal?.throwIfAborted();
293
+ let opCountDiff = 0;
294
+ // Do the CLEAR operation in batches, with each batch a separate transaction.
295
+ // The state after each batch is fully consistent.
296
+ // We need a transaction per batch to make sure checksums stay consistent.
297
+ await session.withTransaction(
298
+ async () => {
299
+ const query = bucketContext.collection.find<Pick<BucketDataDocumentV1, '_id' | CompactClearProperties>>(
300
+ opFilter,
301
+ {
302
+ session,
303
+ sort: { _id: 1 },
304
+ projection: {
305
+ _id: 1,
306
+ op: 1,
307
+ checksum: 1,
308
+ target_op: 1
309
+ },
310
+ limit: this.clearBatchLimit
311
+ }
312
+ );
313
+ let checksum = 0;
314
+ let lastOp: Pick<BucketDataDoc, 'o' | CompactClearProperties> | null = null;
315
+ let targetOp: bigint | null = null;
316
+ let gotAnOp = false;
317
+ let numberOfOpsToClear = 0;
318
+ for await (const rawOp of query.stream()) {
319
+ const op = bucketContext.fromPartialPersistedDocument(rawOp);
320
+
321
+ if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
322
+ checksum = utils.addChecksums(checksum, Number(op.checksum));
323
+ lastOp = op;
324
+ numberOfOpsToClear += 1;
325
+ if (op.op != 'CLEAR') {
326
+ gotAnOp = true;
327
+ }
328
+ if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
329
+ targetOp = op.target_op;
330
+ }
331
+ } else {
332
+ throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
333
+ }
334
+ }
335
+ if (!gotAnOp) {
336
+ done = true;
337
+ return;
338
+ }
339
+
340
+ this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
341
+ await bucketContext.collection.deleteMany(
342
+ {
343
+ _id: {
344
+ $gte: bucketContext.minId,
345
+ $lte: bucketContext.docId(lastOp!.o)
346
+ }
347
+ },
348
+ { session }
349
+ );
350
+
351
+ const op = bucketContext.toPersistedDocument({
352
+ o: lastOp!.o,
353
+ op: 'CLEAR',
354
+ checksum: BigInt(checksum),
355
+ data: null,
356
+ target_op: targetOp
357
+ });
358
+ await bucketContext.collection.insertOne(op, { session });
359
+
360
+ opCountDiff = -numberOfOpsToClear + 1;
361
+ },
362
+ {
363
+ writeConcern: { w: 'majority' },
364
+ readConcern: { level: 'snapshot' }
365
+ }
366
+ );
367
+ // Update outside the transaction, since the transaction can be retried multiple times.
368
+ currentState.opCount += opCountDiff;
369
+ }
370
+ } finally {
371
+ await session.endSession();
372
+ }
373
+ }
374
+
375
+ private formatBucketDataKey(doc: Pick<BucketDataDoc, 'bucketKey' | 'o'>) {
376
+ return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
377
+ }
93
378
  }