@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,9 +1,12 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
3
+ import { addChecksums, utils } from '@powersync/service-core';
3
4
  import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
4
5
  import { MongoCompactor } from '../MongoCompactor.js';
6
+ import { cacheKey } from '../OperationBatch.js';
5
7
  import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
6
8
  export class MongoCompactorV1 extends MongoCompactor {
9
+ updates = [];
7
10
  async *dirtyBucketBatches(options) {
8
11
  if (options.minBucketChanges <= 0) {
9
12
  throw new ReplicationAssertionError('minBucketChanges must be >= 1');
@@ -31,7 +34,7 @@ export class MongoCompactorV1 extends MongoCompactor {
31
34
  return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
32
35
  bucket,
33
36
  end: this.maxOpId
34
- })));
37
+ })), { readOptions: { readConcern: 'snapshot' } });
35
38
  }
36
39
  bucketStateFilter(bucket, _definitionId) {
37
40
  return {
@@ -41,12 +44,258 @@ export class MongoCompactorV1 extends MongoCompactor {
41
44
  }
42
45
  };
43
46
  }
44
- async getBucketDataContext(bucket, _definitionId) {
47
+ getBucketDataContext(bucket) {
45
48
  return new SingleBucketStoreV1(this.db, {
46
49
  replicationStreamId: this.group_id,
47
50
  definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
48
51
  bucket
49
52
  });
50
53
  }
54
+ async compactSingleBucket(bucket) {
55
+ // Do not carry queued writes from a failed attempt into the rescan
56
+ this.updates = [];
57
+ const idLimitBytes = this.idLimitBytes;
58
+ const bucketContext = this.getBucketDataContext(bucket);
59
+ const currentState = {
60
+ bucket,
61
+ definitionId: bucketContext.key.definitionId,
62
+ seen: new Map(),
63
+ trackingSize: 0,
64
+ lastNotPut: null,
65
+ opsSincePut: 0,
66
+ checksum: 0,
67
+ opCount: 0,
68
+ opBytes: 0
69
+ };
70
+ // Constant lower bound.
71
+ const lowerBound = bucketContext.minId;
72
+ // Upper bound is adjusted for each batch.
73
+ let upperBound = bucketContext.maxId;
74
+ // Paginate through bucket data in batches to avoid cursor timeouts.
75
+ while (true) {
76
+ this.signal?.throwIfAborted();
77
+ // Query one batch at a time, to avoid cursor timeouts.
78
+ const pipeline = [
79
+ {
80
+ $match: {
81
+ _id: {
82
+ $gte: lowerBound,
83
+ $lt: upperBound
84
+ }
85
+ }
86
+ },
87
+ { $sort: { _id: -1 } },
88
+ { $limit: this.moveBatchQueryLimit },
89
+ {
90
+ $project: {
91
+ _id: 1,
92
+ op: 1,
93
+ table: 1,
94
+ row_id: 1,
95
+ source_table: 1,
96
+ source_key: 1,
97
+ checksum: 1,
98
+ size: { $bsonSize: '$$ROOT' }
99
+ }
100
+ }
101
+ ];
102
+ const cursor = bucketContext.collection.aggregate(pipeline, {
103
+ // batchSize is 1 more than limit to auto-close the cursor.
104
+ // See https://github.com/mongodb/node-mongodb-native/pull/4580
105
+ batchSize: this.moveBatchQueryLimit + 1
106
+ });
107
+ // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
108
+ // Instead, we load up to the limit.
109
+ const rawBatch = await cursor.toArray();
110
+ const batch = rawBatch.map((document) => {
111
+ const { size, ...rest } = document;
112
+ return {
113
+ doc: bucketContext.fromPersistedDocument(rest),
114
+ size
115
+ };
116
+ });
117
+ if (batch.length == 0) {
118
+ // We've reached the end.
119
+ break;
120
+ }
121
+ // Reuse the exact collection _id value from Mongo for the next bound.
122
+ upperBound = rawBatch[rawBatch.length - 1]._id;
123
+ for (const { doc, size } of batch) {
124
+ if (doc.o > this.maxOpId) {
125
+ continue;
126
+ }
127
+ currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
128
+ currentState.opCount += 1;
129
+ let isPersistentPut = doc.op == 'PUT';
130
+ currentState.opBytes += Number(size);
131
+ if (doc.op == 'REMOVE' || doc.op == 'PUT') {
132
+ const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table, doc.source_key)}`;
133
+ const targetOp = currentState.seen.get(key);
134
+ if (targetOp) {
135
+ // Will convert to MOVE, so don't count as PUT.
136
+ isPersistentPut = false;
137
+ this.updates.push({
138
+ updateOne: {
139
+ filter: { _id: bucketContext.docId(doc.o) },
140
+ update: {
141
+ $set: {
142
+ op: 'MOVE',
143
+ target_op: targetOp
144
+ },
145
+ $unset: {
146
+ source_table: 1,
147
+ source_key: 1,
148
+ table: 1,
149
+ row_id: 1,
150
+ data: 1
151
+ }
152
+ }
153
+ }
154
+ });
155
+ // TODO: better estimate for this.
156
+ currentState.opBytes += 200 - Number(size);
157
+ }
158
+ else if (currentState.trackingSize < idLimitBytes) {
159
+ // flatstr reduces the memory usage by flattening the string.
160
+ currentState.seen.set(utils.flatstr(key), doc.o);
161
+ // length + 16 for the string
162
+ // 24 for the bigint
163
+ // 50 for map overhead
164
+ // 50 for additional overhead
165
+ currentState.trackingSize += key.length + 140;
166
+ }
167
+ }
168
+ if (isPersistentPut) {
169
+ currentState.lastNotPut = null;
170
+ currentState.opsSincePut = 0;
171
+ }
172
+ else if (doc.op != 'CLEAR') {
173
+ if (currentState.lastNotPut == null) {
174
+ currentState.lastNotPut = doc.o;
175
+ }
176
+ currentState.opsSincePut += 1;
177
+ }
178
+ if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
179
+ await this.flush(bucketContext);
180
+ }
181
+ }
182
+ this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
183
+ }
184
+ // Free memory before clearing the bucket.
185
+ currentState.seen.clear();
186
+ if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
187
+ this.logger.info(`Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
188
+ // Need flush() before clear().
189
+ await this.flush(bucketContext);
190
+ await this.clearBucket(currentState, bucketContext);
191
+ }
192
+ // Do this after clearBucket so we have accurate counts.
193
+ this.updateBucketChecksums(currentState, this.maxOpId);
194
+ // Need another flush after updateBucketChecksums().
195
+ await this.flush(bucketContext);
196
+ }
197
+ async flush(bucketContext) {
198
+ if (this.updates.length > 0) {
199
+ this.logger.info(`Compacting ${this.updates.length} ops`);
200
+ await bucketContext.collection.bulkWrite(this.updates, {
201
+ // Order is not important. Since checksums are not affected, these operations can happen in any order,
202
+ // and it's fine if the operations are partially applied. Each individual operation is atomic.
203
+ ordered: false
204
+ });
205
+ this.updates = [];
206
+ }
207
+ await this.flushBucketStateUpdates();
208
+ }
209
+ /**
210
+ * Perform a CLEAR compact for a bucket.
211
+ *
212
+ * @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
213
+ */
214
+ async clearBucket(currentState, bucketContext) {
215
+ const clearOp = currentState.lastNotPut;
216
+ const opFilter = {
217
+ _id: {
218
+ $gte: bucketContext.minId,
219
+ $lte: bucketContext.docId(clearOp)
220
+ }
221
+ };
222
+ const session = this.db.client.startSession();
223
+ try {
224
+ let done = false;
225
+ while (!done) {
226
+ this.signal?.throwIfAborted();
227
+ let opCountDiff = 0;
228
+ // Do the CLEAR operation in batches, with each batch a separate transaction.
229
+ // The state after each batch is fully consistent.
230
+ // We need a transaction per batch to make sure checksums stay consistent.
231
+ await session.withTransaction(async () => {
232
+ const query = bucketContext.collection.find(opFilter, {
233
+ session,
234
+ sort: { _id: 1 },
235
+ projection: {
236
+ _id: 1,
237
+ op: 1,
238
+ checksum: 1,
239
+ target_op: 1
240
+ },
241
+ limit: this.clearBatchLimit
242
+ });
243
+ let checksum = 0;
244
+ let lastOp = null;
245
+ let targetOp = null;
246
+ let gotAnOp = false;
247
+ let numberOfOpsToClear = 0;
248
+ for await (const rawOp of query.stream()) {
249
+ const op = bucketContext.fromPartialPersistedDocument(rawOp);
250
+ if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
251
+ checksum = utils.addChecksums(checksum, Number(op.checksum));
252
+ lastOp = op;
253
+ numberOfOpsToClear += 1;
254
+ if (op.op != 'CLEAR') {
255
+ gotAnOp = true;
256
+ }
257
+ if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
258
+ targetOp = op.target_op;
259
+ }
260
+ }
261
+ else {
262
+ throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
263
+ }
264
+ }
265
+ if (!gotAnOp) {
266
+ done = true;
267
+ return;
268
+ }
269
+ this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
270
+ await bucketContext.collection.deleteMany({
271
+ _id: {
272
+ $gte: bucketContext.minId,
273
+ $lte: bucketContext.docId(lastOp.o)
274
+ }
275
+ }, { session });
276
+ const op = bucketContext.toPersistedDocument({
277
+ o: lastOp.o,
278
+ op: 'CLEAR',
279
+ checksum: BigInt(checksum),
280
+ data: null,
281
+ target_op: targetOp
282
+ });
283
+ await bucketContext.collection.insertOne(op, { session });
284
+ opCountDiff = -numberOfOpsToClear + 1;
285
+ }, {
286
+ writeConcern: { w: 'majority' },
287
+ readConcern: { level: 'snapshot' }
288
+ });
289
+ // Update outside the transaction, since the transaction can be retried multiple times.
290
+ currentState.opCount += opCountDiff;
291
+ }
292
+ }
293
+ finally {
294
+ await session.endSession();
295
+ }
296
+ }
297
+ formatBucketDataKey(doc) {
298
+ return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
299
+ }
51
300
  }
52
301
  //# sourceMappingURL=MongoCompactorV1.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAI9E,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,aAAwC;QAExC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAW,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGvE,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAmC,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACvF,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAK/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK1C,OAAO,GAAwD,EAAE,CAAC;IAEnE,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,EACH,EAAE,WAAW,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,CAC7C,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,MAAc;QACzC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAChD,mEAAmE;QACnE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,YAAY,GAAuB;YACvC,MAAM;YACN,YAAY,EAAE,aAAa,CAAC,GAAG,CAAC,YAAY;YAC5C,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,CAAC;QAEF,wBAAwB;QACxB,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QACvC,0CAA0C;QAC1C,IAAI,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QAErC,oEAAoE;QACpE,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,uDAAuD;YACvD,MAAM,QAAQ,GAAG;gBACf;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,SAAS,CAAmD,QAAQ,EAAE;gBAC5G,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,mBAAmB,GAAG,CAAC;aACxC,CAAC,CAAC;YACH,+GAA+G;YAC/G,oCAAoC;YACpC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACtC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;gBACnC,OAAO;oBACL,GAAG,EAAE,aAAa,CAAC,qBAAqB,CAAC,IAAI,CAAC;oBAC9C,IAAI;iBACL,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,yBAAyB;gBACzB,MAAM;YACR,CAAC;YAED,sEAAsE;YACtE,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAE/C,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,+CAA+C;wBAC/C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gCAC3C,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACiD;6BACrD;yBACF,CAAC,CAAC;wBAEH,kCAAkC;wBAClC,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC7C,CAAC;yBAAM,IAAI,YAAY,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC;wBACpD,6DAA6D;wBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;wBACjD,6BAA6B;wBAC7B,oBAAoB;wBACpB,sBAAsB;wBACtB,6BAA6B;wBAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;oBAChD,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC;oBAClC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,MAAM,oBAAoB,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,0CAA0C;QAC1C,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sBAAsB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CAC5H,CAAC;YACF,+BAA+B;YAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACtD,CAAC;QAED,wDAAwD;QACxD,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,oDAAoD;QACpD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,aAAkC;QACpD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YAC1D,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBACrD,sGAAsG;gBACtG,8FAA8F;gBAC9F,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC,EAAE,aAAkC;QAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE,aAAa,CAAC,KAAK;gBACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;aACnC;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;gBAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CACzC,QAAQ,EACR;wBACE,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CACF,CAAC;oBACF,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,MAAM,GAA6D,IAAI,CAAC;oBAC5E,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACzC,MAAM,EAAE,GAAG,aAAa,CAAC,4BAA4B,CAAC,KAAK,CAAC,CAAC;wBAE7D,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,MAAM,GAAG,EAAE,CAAC;4BACZ,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;gCAC1E,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;4BAC1B,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1G,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;oBACjF,MAAM,aAAa,CAAC,UAAU,CAAC,UAAU,CACvC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE,aAAa,CAAC,KAAK;4BACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC;yBACrC;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;wBAC3C,CAAC,EAAE,MAAO,CAAC,CAAC;wBACZ,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,CAAC,CAAC;oBACH,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;oBAE1D,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,uFAAuF;gBACvF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,GAA2C;QACrE,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,mBAAmB,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;IACjF,CAAC;CACF"}
@@ -1,41 +1,40 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage, utils } from '@powersync/service-core';
2
+ import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
3
3
  import { ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
4
4
  import { MongoBucketStorage } from '../../MongoBucketStorage.js';
5
- import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js';
6
- import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
5
+ import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
7
6
  import { MongoChecksums } from '../MongoChecksums.js';
8
7
  import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
9
8
  import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
10
- import { MongoPersistedSyncRulesContentV1 } from '../MongoPersistedSyncRulesContent.js';
9
+ import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
11
10
  import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
12
11
  import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
13
12
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
13
+ export interface MongoSyncBucketStorageContextV1 {
14
+ db: VersionedPowerSyncMongoV1;
15
+ replicationStreamId: number;
16
+ readPreference?: mongo.ReadPreference;
17
+ }
14
18
  export declare class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
15
19
  readonly db: VersionedPowerSyncMongoV1;
16
20
  readonly checksums: MongoChecksumsV1;
17
- constructor(factory: MongoBucketStorage, group_id: number, sync_rules: MongoPersistedSyncRulesContentV1, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
21
+ constructor(factory: MongoBucketStorage, replicationStreamId: number, replicationStream: MongoPersistedReplicationStream, replicationStreamName: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
18
22
  protected initializeVersionStorage(): Promise<void>;
19
- protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
23
+ protected fetchPersistedOpHead(): Promise<InternalOpId | null>;
24
+ protected createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
20
25
  protected fetchCheckpointState(session: mongo.ClientSession): Promise<{
21
26
  checkpoint: bigint;
22
27
  lsn: string | null;
23
28
  } | null>;
24
- protected getWriterSyncState(): Promise<{
25
- lastCheckpointLsn: string | null;
26
- resumeFromLsn: string | null;
27
- keepaliveOp: bigint | null;
28
- syncConfigId: null;
29
- }>;
30
29
  protected terminateSyncRuleState(): Promise<void>;
31
- protected getStatusImpl(): Promise<storage.SyncRuleStatus>;
30
+ protected getStatusImpl(): Promise<storage.ReplicationStreamStatus>;
32
31
  protected clearSyncRuleState(): Promise<void>;
33
32
  protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
34
33
  createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
35
34
  protected createMongoParameterCompactor(checkpoint: InternalOpId, options: storage.CompactOptions): MongoParameterCompactor;
36
- protected get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>;
35
+ protected get versionContext(): MongoSyncBucketStorageContextV1;
37
36
  protected getParameterSetsImpl(checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
38
- protected getBucketDataBatchImpl(checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
37
+ protected getBucketDataBatchImpl(checkpoint: MongoSyncBucketStorageCheckpoint, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
39
38
  protected clearBucketData(signal?: AbortSignal): Promise<void>;
40
39
  protected clearParameterIndexes(signal?: AbortSignal): Promise<void>;
41
40
  protected clearSourceRecords(signal?: AbortSignal): Promise<void>;
@@ -44,7 +43,7 @@ export declare class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
44
43
  protected getDataBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
45
44
  protected getParameterBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
46
45
  }
47
- export declare function getParameterSetsV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
48
- export declare function getBucketDataBatchV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
49
- export declare function getDataBucketChangesV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
50
- export declare function getParameterBucketChangesV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
46
+ export declare function getParameterSetsV1(ctx: MongoSyncBucketStorageContextV1, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
47
+ export declare function getBucketDataBatchV1(ctx: MongoSyncBucketStorageContextV1, checkpoint: MongoSyncBucketStorageCheckpoint, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
48
+ export declare function getDataBucketChangesV1(ctx: MongoSyncBucketStorageContextV1, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
49
+ export declare function getParameterBucketChangesV1(ctx: MongoSyncBucketStorageContextV1, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
@@ -3,7 +3,7 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework';
3
3
  import { deserializeParameterLookup, internalToExternalOpId, maxLsn, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
4
4
  import { JSONBig } from '@powersync/service-jsonbig';
5
5
  import * as bson from 'bson';
6
- import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
6
+ import { clearCollectionInIdBatches, clearCollectionInIdRanges, idPrefixFilter, mapOpEntry, readSingleBatch, retryOnMongoMaxTimeMSExpired, setSessionSnapshotTime } from '../../../utils/util.js';
7
7
  import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js';
8
8
  import { loadBucketDataDocumentV1 } from './models.js';
9
9
  import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js';
@@ -11,15 +11,34 @@ import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
11
11
  import { MongoCompactorV1 } from './MongoCompactorV1.js';
12
12
  import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js';
13
13
  export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
14
- constructor(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options) {
15
- super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
14
+ constructor(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
15
+ super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
16
16
  }
17
17
  async initializeVersionStorage() { }
18
- createWriterImpl(batchOptions) {
19
- return new MongoBucketBatchV1(batchOptions);
18
+ async fetchPersistedOpHead() {
19
+ const doc = (await this.db.sync_rules.findOne({ _id: this.replicationStreamId }, { projection: { keepalive_op: 1, last_checkpoint: 1 } }));
20
+ // keepalive_op covers ops not yet in a checkpoint (cleared once checkpointed),
21
+ // so the head is the max of the two.
22
+ const keepaliveOp = doc?.keepalive_op == null ? null : BigInt(doc.keepalive_op);
23
+ const lastCheckpoint = doc?.last_checkpoint ?? null;
24
+ if (keepaliveOp == null && lastCheckpoint == null) {
25
+ return null;
26
+ }
27
+ return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint;
28
+ }
29
+ async createWriterImpl(options) {
30
+ const doc = (await this.db.sync_rules.findOne({
31
+ _id: this.replicationStreamId
32
+ }, { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }));
33
+ return new MongoBucketBatchV1({
34
+ ...this.writerBatchOptions(options),
35
+ // Resume from the last consistent checkpoint, or the in-progress snapshot position if it is newer.
36
+ resumeFromLsn: maxLsn(doc?.last_checkpoint_lsn, doc?.snapshot_lsn),
37
+ keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null
38
+ });
20
39
  }
21
40
  async fetchCheckpointState(session) {
22
- const doc = (await this.db.sync_rules.findOne({ _id: this.group_id }, {
41
+ const doc = (await this.db.sync_rules.findOne({ _id: this.replicationStreamId }, {
23
42
  session,
24
43
  projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
25
44
  }));
@@ -31,21 +50,9 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
31
50
  lsn: doc.last_checkpoint_lsn ?? null
32
51
  };
33
52
  }
34
- async getWriterSyncState() {
35
- const doc = (await this.db.sync_rules.findOne({
36
- _id: this.group_id
37
- }, { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }));
38
- const checkpointLsn = doc?.last_checkpoint_lsn ?? null;
39
- return {
40
- lastCheckpointLsn: checkpointLsn,
41
- resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
42
- keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
43
- syncConfigId: null
44
- };
45
- }
46
53
  async terminateSyncRuleState() {
47
54
  await this.db.sync_rules.updateOne({
48
- _id: this.group_id
55
+ _id: this.replicationStreamId
49
56
  }, {
50
57
  $set: {
51
58
  state: storage.SyncRuleState.TERMINATED,
@@ -56,30 +63,25 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
56
63
  }
57
64
  async getStatusImpl() {
58
65
  const doc = (await this.db.sync_rules.findOne({
59
- _id: this.group_id
66
+ _id: this.replicationStreamId
60
67
  }, {
61
68
  projection: {
62
69
  snapshot_done: 1,
63
70
  last_checkpoint_lsn: 1,
64
- state: 1,
65
- snapshot_lsn: 1,
66
- keepalive_op: 1
71
+ snapshot_lsn: 1
67
72
  }
68
73
  }));
69
74
  if (doc == null) {
70
75
  throw new ServiceAssertionError('Cannot find replication stream status');
71
76
  }
72
77
  return {
73
- snapshot_done: doc.snapshot_done,
74
- snapshot_lsn: doc.snapshot_lsn ?? null,
75
- active: doc.state == storage.SyncRuleState.ACTIVE,
76
- checkpoint_lsn: doc.last_checkpoint_lsn,
77
- keepalive_op: doc.keepalive_op == null ? null : BigInt(doc.keepalive_op)
78
+ snapshotDone: doc.snapshot_done && doc.last_checkpoint_lsn != null,
79
+ resumeLsn: maxLsn(doc.snapshot_lsn, doc.last_checkpoint_lsn)
78
80
  };
79
81
  }
80
82
  async clearSyncRuleState() {
81
83
  await this.db.sync_rules.updateOne({
82
- _id: this.group_id
84
+ _id: this.replicationStreamId
83
85
  }, {
84
86
  $set: {
85
87
  snapshot_done: false,
@@ -94,23 +96,23 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
94
96
  }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS });
95
97
  }
96
98
  createMongoChecksums(options) {
97
- return new MongoChecksumsV1(this.db, this.group_id, {
99
+ return new MongoChecksumsV1(this.db, this.replicationStreamId, {
98
100
  ...options.checksumOptions,
99
- storageConfig: options?.storageConfig,
100
- mapping: this.sync_rules.mapping
101
+ checksumCacheTtlMs: options.checksumCacheTtlMs,
102
+ storageConfig: options?.storageConfig
101
103
  });
102
104
  }
103
105
  createMongoCompactor(options) {
104
106
  return new MongoCompactorV1(this, this.db, options);
105
107
  }
106
108
  createMongoParameterCompactor(checkpoint, options) {
107
- return new MongoParameterCompactorV1(this.db, this.group_id, checkpoint, options);
109
+ return new MongoParameterCompactorV1(this.db, this.replicationStreamId, checkpoint, options);
108
110
  }
109
111
  get versionContext() {
110
112
  return {
111
113
  db: this.db,
112
- group_id: this.group_id,
113
- mapping: this.mapping
114
+ replicationStreamId: this.replicationStreamId,
115
+ readPreference: this.readPreference
114
116
  };
115
117
  }
116
118
  getParameterSetsImpl(checkpoint, lookups, limit) {
@@ -120,29 +122,37 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
120
122
  return getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options);
121
123
  }
122
124
  async clearBucketData(signal) {
123
- await this.clearDeleteMany('bucket data', () => this.db.bucket_data.deleteMany({
124
- _id: idPrefixFilter({ g: this.group_id }, ['b', 'o'])
125
- }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
125
+ await clearCollectionInIdRanges(this.logger, 'bucket data', this.db.bucket_data, {
126
+ _id: idPrefixFilter({ g: this.replicationStreamId }, ['b', 'o'])
127
+ }, signal, this.clearBatchThrottleRate);
126
128
  }
127
129
  async clearParameterIndexes(signal) {
128
- await this.clearDeleteMany('parameter index', () => this.db.parameterIndexV1.deleteMany({
129
- 'key.g': this.group_id
130
- }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
130
+ await clearCollectionInIdBatches(this.logger, 'parameter index', this.db.parameterIndexV1, {
131
+ 'key.g': this.replicationStreamId
132
+ }, signal, this.clearBatchThrottleRate);
131
133
  }
132
134
  async clearSourceRecords(signal) {
133
- await this.clearDeleteMany('source records', () => this.db.sourceRecordsV1.deleteMany({
134
- _id: idPrefixFilter({ g: this.group_id }, ['t', 'k'])
135
- }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
135
+ await clearCollectionInIdRanges(this.logger, 'source records', this.db.sourceRecordsV1, {
136
+ _id: idPrefixFilter({ g: this.replicationStreamId }, ['t', 'k'])
137
+ }, signal, this.clearBatchThrottleRate);
136
138
  }
137
139
  async clearBucketState(signal) {
138
- await this.clearDeleteMany('bucket state', () => this.db.bucketStateV1.deleteMany({
139
- _id: idPrefixFilter({ g: this.group_id }, ['b'])
140
- }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
140
+ await clearCollectionInIdRanges(this.logger, 'bucket state', this.db.bucketStateV1, {
141
+ _id: idPrefixFilter({ g: this.replicationStreamId }, ['b'])
142
+ }, signal, this.clearBatchThrottleRate);
141
143
  }
142
144
  async clearSourceTables(signal) {
143
- await this.clearDeleteMany('source tables', () => this.db.commonSourceTables(this.group_id).deleteMany({
144
- group_id: this.group_id
145
- }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), signal);
145
+ await retryOnMongoMaxTimeMSExpired(() => this.db.sourceTablesV1(this.replicationStreamId).deleteMany({
146
+ group_id: this.replicationStreamId
147
+ }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), {
148
+ signal,
149
+ abortMessage: 'Aborted clearing data',
150
+ // This is a fairly long delay - only expected to hit this when the storage database is under high load.
151
+ retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
152
+ onRetry: () => {
153
+ this.logger.info(`Clearing batch of source tables timed out after ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, retrying...`);
154
+ }
155
+ });
146
156
  }
147
157
  getDataBucketChangesImpl(options) {
148
158
  return getDataBucketChangesV1(this.versionContext, options);
@@ -161,7 +171,7 @@ export async function getParameterSetsV1(ctx, checkpoint, lookups, limit) {
161
171
  .aggregate([
162
172
  {
163
173
  $match: {
164
- 'key.g': ctx.group_id,
174
+ 'key.g': ctx.replicationStreamId,
165
175
  lookup: { $in: lookupFilter },
166
176
  _id: { $lte: checkpoint.checkpoint }
167
177
  }
@@ -217,22 +227,32 @@ export async function* getBucketDataBatchV1(ctx, checkpoint, dataBuckets, option
217
227
  if (dataBuckets.length == 0) {
218
228
  return;
219
229
  }
230
+ const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
231
+ const readConcern = ctx.readPreference == null ? undefined : 'majority';
232
+ const session = readPreference == null || checkpoint.snapshotTime == null
233
+ ? undefined
234
+ : ctx.db.client.startSession({ causalConsistency: true });
235
+ await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
236
+ if (session != null) {
237
+ session.advanceOperationTime(checkpoint.snapshotTime);
238
+ session.advanceClusterTime(checkpoint.clusterTime);
239
+ }
220
240
  let filters = [];
221
241
  const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
222
- if (checkpoint == null) {
242
+ if (checkpoint.checkpoint == null) {
223
243
  throw new Error('checkpoint is null');
224
244
  }
225
- const end = checkpoint;
245
+ const end = checkpoint.checkpoint;
226
246
  for (let { bucket: name, start } of dataBuckets) {
227
247
  filters.push({
228
248
  _id: {
229
249
  $gt: {
230
- g: ctx.group_id,
250
+ g: ctx.replicationStreamId,
231
251
  b: name,
232
252
  o: start
233
253
  },
234
254
  $lte: {
235
- g: ctx.group_id,
255
+ g: ctx.replicationStreamId,
236
256
  b: name,
237
257
  o: end
238
258
  }
@@ -241,10 +261,14 @@ export async function* getBucketDataBatchV1(ctx, checkpoint, dataBuckets, option
241
261
  }
242
262
  const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
243
263
  const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
264
+ // raw: true returns Buffers, but the driver typing doesn't reflect that
265
+ // without an explicit cast to FindCursor<Buffer>.
244
266
  const cursor = ctx.db.bucket_data.find({
245
267
  $or: filters
246
268
  }, {
247
- session: undefined,
269
+ session,
270
+ readPreference,
271
+ readConcern,
248
272
  sort: { _id: 1 },
249
273
  limit: batchLimit,
250
274
  batchSize: batchLimit + 1,
@@ -310,7 +334,7 @@ export async function getDataBucketChangesV1(ctx, options) {
310
334
  const limit = 1000;
311
335
  const bucketStateUpdates = await ctx.db.bucketStateV1
312
336
  .find({
313
- '_id.g': ctx.group_id,
337
+ '_id.g': ctx.replicationStreamId,
314
338
  last_op: { $gt: options.lastCheckpoint.checkpoint }
315
339
  }, {
316
340
  projection: {
@@ -333,7 +357,7 @@ export async function getParameterBucketChangesV1(ctx, options) {
333
357
  const parameterUpdates = await ctx.db.parameterIndexV1
334
358
  .find({
335
359
  _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
336
- 'key.g': ctx.group_id
360
+ 'key.g': ctx.replicationStreamId
337
361
  }, {
338
362
  projection: {
339
363
  lookup: 1