@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,32 +1,104 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
3
- import { MongoCompactor } from '../MongoCompactor.js';
4
- import { SingleBucketStoreV3 } from './SingleBucketStoreV3.js';
2
+ import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
3
+ import { addChecksums, utils } from '@powersync/service-core';
4
+ import { ConcurrentCompactionError, MongoCompactor } from '../MongoCompactor.js';
5
+ import { cacheKey } from '../OperationBatch.js';
6
+ import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js';
7
+ import { BucketDataContextV3 } from './BucketDataContextV3.js';
8
+ import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js';
9
+ import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
10
+ import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
11
+ /**
12
+ * Read one bounded prefix from a descending compaction cursor.
13
+ *
14
+ * The document that would cross the byte limit is deliberately not returned:
15
+ * pagination resumes below the last returned `_id`, so that document remains
16
+ * eligible for the next query. The first document is always accepted to ensure
17
+ * progress when a single document exceeds the configured byte limit.
18
+ *
19
+ * `hasMore` is conservative when the document limit is reached. An extra empty
20
+ * query is preferable to exhausting the cursor just to determine whether the
21
+ * limited MongoDB query contained another document.
22
+ */
23
+ async function readCompactionBatch(cursor, options) {
24
+ const documents = [];
25
+ let cumulativeBytes = 0;
26
+ try {
27
+ for await (const document of cursor) {
28
+ if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) {
29
+ return { documents, hasMore: true };
30
+ }
31
+ documents.push(document);
32
+ cumulativeBytes += document.size;
33
+ if (documents.length >= options.documentLimit) {
34
+ return { documents, hasMore: true };
35
+ }
36
+ }
37
+ return { documents, hasMore: false };
38
+ }
39
+ finally {
40
+ await cursor.close();
41
+ }
42
+ }
5
43
  export class MongoCompactorV3 extends MongoCompactor {
44
+ async compact() {
45
+ if (this.storage.objectStorage) {
46
+ // Clean these before compacting - should be quick in most cases.
47
+ try {
48
+ await this.objectStorageLifecycle.cleanup(this.logger);
49
+ }
50
+ catch (e) {
51
+ // In this case, still continue normal compact process
52
+ this.logger.error(`Failed to clean up object storage deletion markers before compaction`, e);
53
+ }
54
+ }
55
+ await super.compact();
56
+ if (this.storage.objectStorage) {
57
+ // Cleanup for any produced during compacting.
58
+ // Note that markers only expire after a delay, so this may skip many produced during this compact
59
+ // run. However, during long compact runs, this may also have many ones it can clean up.
60
+ await this.objectStorageLifecycle.cleanup(this.logger);
61
+ }
62
+ }
63
+ get objectStorageLifecycle() {
64
+ if (!this.storage.objectStorage) {
65
+ throw new Error('Object storage is not configured');
66
+ }
67
+ return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage);
68
+ }
6
69
  async *dirtyBucketBatches(options) {
7
70
  if (options.minBucketChanges <= 0) {
8
71
  throw new ReplicationAssertionError('minBucketChanges must be >= 1');
9
72
  }
10
- // Same scan strategy as V1, but with the V3 bucket_state key shape.
11
- yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV3(this.group_id), { d: new mongo.MinKey(), b: new mongo.MinKey() }, { d: new mongo.MaxKey(), b: new mongo.MaxKey() }, options, (bucketState) => bucketState._id.d);
73
+ const collection = this.db.bucketState(this.group_id);
74
+ yield* this.dirtyBucketBatchesForCollection(collection, { d: new mongo.MinKey(), b: new mongo.MinKey() }, { d: new mongo.MaxKey(), b: new mongo.MaxKey() }, options, (bucketState) => bucketState._id.d);
12
75
  }
13
76
  async dirtyBucketBatchForChecksums(options) {
14
77
  if (options.minBucketChanges <= 0) {
15
78
  throw new ReplicationAssertionError('minBucketChanges must be >= 1');
16
79
  }
17
- return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketStateV3(this.group_id), {
80
+ return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketState(this.group_id), {
18
81
  'estimate_since_compact.count': { $gte: options.minBucketChanges }
19
82
  }, (bucketState) => bucketState._id.d);
20
83
  }
21
84
  async writeBucketStateUpdates() {
22
85
  await this.db
23
- .bucketStateV3(this.group_id)
24
- .bulkWrite(this.bucketStateUpdates, { ordered: false });
86
+ .bucketState(this.group_id)
87
+ .bulkWrite(this.bucketStateUpdates, {
88
+ ordered: false
89
+ });
90
+ }
91
+ /**
92
+ * The compactor operates on persisted definition ids only - never on parsed sources.
93
+ * This narrowed view makes the source-resolving checksum methods unreachable here.
94
+ */
95
+ get definitionChecksums() {
96
+ return this.storage.checksums;
25
97
  }
26
98
  async computeChecksumsForBuckets(buckets) {
27
- return this.storage.checksums.computePartialChecksumsDirectByDefinition(buckets.map(({ bucket, definitionId }) => {
99
+ return this.definitionChecksums.computePartialChecksumsDirectByDefinition(buckets.map(({ bucket, definitionId }) => {
28
100
  if (definitionId == null) {
29
- throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`);
101
+ throw new ServiceAssertionError(`Missing definitionId for bucket checksum update on bucket ${bucket}`);
30
102
  }
31
103
  return {
32
104
  bucket,
@@ -47,22 +119,574 @@ export class MongoCompactorV3 extends MongoCompactor {
47
119
  };
48
120
  }
49
121
  async getBucketDataContext(bucket, definitionId) {
50
- if (definitionId == null) {
51
- // Not the _most_ efficient approach, but this is not used often
52
- const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds();
53
- if (allDefinitionIds.length == 0) {
54
- return null;
55
- }
56
- const potentialIds = allDefinitionIds.map((definitionId) => ({ d: definitionId, b: bucket }));
57
- const bucketState = await this.db.bucketStateV3(this.group_id).findOne({
58
- _id: { $in: potentialIds }
122
+ let resolvedDefinitionId = definitionId;
123
+ if (resolvedDefinitionId == null) {
124
+ const allDefinitionIds = this.storage.storageIds.bucketDefinitionIds;
125
+ if (allDefinitionIds.length > 0) {
126
+ const potentialIds = allDefinitionIds.map((id) => ({ d: id, b: bucket }));
127
+ const bucketState = await this.db.bucketState(this.group_id).findOne({
128
+ _id: { $in: potentialIds }
129
+ });
130
+ if (bucketState != null) {
131
+ resolvedDefinitionId = bucketState._id.d;
132
+ }
133
+ }
134
+ }
135
+ if (resolvedDefinitionId == null) {
136
+ return null;
137
+ }
138
+ return new BucketDataContextV3(this.db, {
139
+ bucket,
140
+ definitionId: resolvedDefinitionId,
141
+ replicationStreamId: this.group_id
142
+ });
143
+ }
144
+ async compactSingleBucket(bucket, definitionId = null) {
145
+ const bucketContext = await this.getBucketDataContext(bucket, definitionId);
146
+ if (bucketContext == null) {
147
+ return;
148
+ }
149
+ const resolvedDefinitionId = bucketContext.key.definitionId;
150
+ const collection = this.db.bucketData(this.group_id, resolvedDefinitionId);
151
+ const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId };
152
+ const lowerBound = bucketContext.minId;
153
+ let upperBound = bucketContext.docId(this.maxOpId + 1n);
154
+ let totalChecksum = 0;
155
+ let totalOpCount = 0;
156
+ let totalOpBytes = 0;
157
+ let lastNotPut = null;
158
+ let opsSincePut = 0;
159
+ let compactedOpId = null;
160
+ let clearBoundary = null;
161
+ const seen = new Map();
162
+ let trackingSize = 0;
163
+ let pendingGroup = null;
164
+ // --- Read batch from MongoDB ---
165
+ while (true) {
166
+ this.signal?.throwIfAborted();
167
+ const pipeline = [
168
+ {
169
+ $match: {
170
+ _id: {
171
+ $gte: lowerBound,
172
+ $lt: upperBound
173
+ }
174
+ }
175
+ },
176
+ { $sort: { _id: -1 } },
177
+ { $limit: this.moveBatchQueryLimit },
178
+ {
179
+ $project: {
180
+ _id: 1,
181
+ min_op: 1,
182
+ checksum: 1,
183
+ count: 1,
184
+ size: 1,
185
+ target_op: 1,
186
+ ops: 1,
187
+ storage_ref: 1
188
+ }
189
+ }
190
+ ];
191
+ const batch = await readCompactionBatch(collection.aggregate(pipeline, {
192
+ batchSize: this.moveBatchQueryLimit + 1
193
+ }), {
194
+ byteLimit: this.moveBatchByteLimit,
195
+ documentLimit: this.moveBatchQueryLimit
196
+ });
197
+ const batchDocs = batch.documents;
198
+ if (batchDocs.length == 0) {
199
+ // No more documents in this bucket — compaction complete.
200
+ break;
201
+ }
202
+ await hydrateBucketDataDocuments(batchDocs, this.storage.objectStorage, { signal: this.signal });
203
+ // Compact each document independently, then greedily merge adjacent
204
+ // post-compaction results. This preserves existing boundaries unless
205
+ // merging is useful, and writes each final object at most once.
206
+ for (const doc of batchDocs) {
207
+ compactedOpId ??= doc._id.o;
208
+ const originalOps = Array.from(loadBucketDataDocument(context, doc));
209
+ let changed = false;
210
+ const compactedOps = [];
211
+ let maxTargetOp = doc.target_op ?? null;
212
+ for (let index = originalOps.length - 1; index >= 0; index--) {
213
+ const op = originalOps[index];
214
+ if (op.op == 'PUT' || op.op == 'REMOVE') {
215
+ const key = `${op.table}/${op.row_id}/${cacheKey(op.source_table, op.source_key)}`;
216
+ const targetOp = seen.get(key);
217
+ if (targetOp != null) {
218
+ maxTargetOp = maxOpId(maxTargetOp, targetOp);
219
+ compactedOps.push({
220
+ ...op,
221
+ op: 'MOVE',
222
+ table: undefined,
223
+ row_id: undefined,
224
+ source_table: undefined,
225
+ source_key: undefined,
226
+ data: null
227
+ });
228
+ changed = true;
229
+ if (lastNotPut == null) {
230
+ lastNotPut = op.o;
231
+ }
232
+ opsSincePut += 1;
233
+ }
234
+ else {
235
+ if (trackingSize < this.idLimitBytes) {
236
+ seen.set(utils.flatstr(key), op.o);
237
+ trackingSize += key.length + 140;
238
+ }
239
+ compactedOps.push(op);
240
+ if (op.op == 'PUT') {
241
+ lastNotPut = null;
242
+ opsSincePut = 0;
243
+ }
244
+ else {
245
+ if (lastNotPut == null) {
246
+ lastNotPut = op.o;
247
+ }
248
+ opsSincePut += 1;
249
+ }
250
+ }
251
+ }
252
+ else {
253
+ compactedOps.push(op);
254
+ if (op.op != 'CLEAR') {
255
+ if (lastNotPut == null) {
256
+ lastNotPut = op.o;
257
+ }
258
+ opsSincePut += 1;
259
+ }
260
+ }
261
+ }
262
+ compactedOps.reverse();
263
+ for (const op of compactedOps) {
264
+ totalChecksum = addChecksums(totalChecksum, Number(op.checksum));
265
+ totalOpBytes += op.data?.length ?? 0;
266
+ }
267
+ totalOpCount += compactedOps.length;
268
+ const candidate = {
269
+ inputs: [doc],
270
+ ops: compactedOps,
271
+ changed,
272
+ targetOp: maxTargetOp
273
+ };
274
+ if (pendingGroup == null) {
275
+ pendingGroup = candidate;
276
+ }
277
+ else {
278
+ const mergedOps = [...candidate.ops, ...pendingGroup.ops];
279
+ const mergedSize = serializeBucketData(bucket, mergedOps, { targetOp: maxTargetOp }).size;
280
+ if (mergedSize <= DEFAULT_MAX_DOC_SIZE_BYTES) {
281
+ pendingGroup = {
282
+ inputs: [...candidate.inputs, ...pendingGroup.inputs],
283
+ ops: mergedOps,
284
+ changed: candidate.changed || pendingGroup.changed,
285
+ targetOp: maxOpId(maxTargetOp, pendingGroup.targetOp)
286
+ };
287
+ }
288
+ else {
289
+ const flushedGroup = pendingGroup;
290
+ const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, context);
291
+ if (lastNotPut != null &&
292
+ flushedGroup.ops[0].o <= lastNotPut &&
293
+ flushedGroup.ops[flushedGroup.ops.length - 1].o >= lastNotPut) {
294
+ clearBoundary = { opId: lastNotPut, documentId };
295
+ }
296
+ pendingGroup = candidate;
297
+ }
298
+ }
299
+ }
300
+ // --- Advance to next batch ---
301
+ upperBound = batchDocs[batchDocs.length - 1]._id;
302
+ if (!batch.hasMore) {
303
+ break;
304
+ }
305
+ this.logger.info(`Compacted batch of ${batchDocs.length} documents for bucket ${bucket}`);
306
+ }
307
+ if (pendingGroup != null) {
308
+ const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, context);
309
+ if (lastNotPut != null &&
310
+ pendingGroup.ops[0].o <= lastNotPut &&
311
+ pendingGroup.ops[pendingGroup.ops.length - 1].o >= lastNotPut) {
312
+ clearBoundary = { opId: lastNotPut, documentId };
313
+ }
314
+ }
315
+ if (compactedOpId == null) {
316
+ return;
317
+ }
318
+ // --- Clear: collapse leading MOVE/REMOVE/CLEAR sequence ---
319
+ if (lastNotPut != null && opsSincePut >= 2) {
320
+ if (clearBoundary == null || clearBoundary.opId != lastNotPut) {
321
+ throw new ReplicationAssertionError(`Missing CLEAR boundary document for bucket ${bucket}`);
322
+ }
323
+ totalOpCount += await this.clearBucketLeading(lastNotPut, clearBoundary.documentId, bucketContext, collection, context);
324
+ }
325
+ // --- Finalize: update bucket checksums and state ---
326
+ this.updateBucketChecksums({
327
+ bucket,
328
+ definitionId: resolvedDefinitionId,
329
+ seen: new Map(),
330
+ trackingSize: 0,
331
+ lastNotPut: lastNotPut,
332
+ opsSincePut: opsSincePut,
333
+ checksum: totalChecksum,
334
+ opCount: totalOpCount,
335
+ opBytes: totalOpBytes
336
+ }, compactedOpId);
337
+ if (this.bucketStateUpdates.length > 0) {
338
+ await this.writeBucketStateUpdates();
339
+ this.bucketStateUpdates = [];
340
+ }
341
+ logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`);
342
+ }
343
+ /**
344
+ * Persist replacement objects before starting the transaction, then atomically
345
+ * publish their lifecycle markers alongside the MongoDB document replacement.
346
+ * If verification or the transaction fails, the prepared markers retain enough
347
+ * information for the uploaded objects to be cleaned up later.
348
+ */
349
+ async flushCompactionGroup(bucket, group, bucketContext, context) {
350
+ if (group.inputs.length == 1 && !group.changed) {
351
+ return group.inputs[0]._id;
352
+ }
353
+ const inputs = group.inputs;
354
+ const idsToDelete = inputs.map((doc) => doc._id);
355
+ const expectedDocCount = inputs.length;
356
+ const expectedChecksum = inputs.reduce((sum, doc) => sum + doc.checksum, 0n);
357
+ const expectedOpCount = inputs.reduce((sum, doc) => sum + doc.count, 0);
358
+ const oldStoragePaths = inputs.flatMap((doc) => (doc.storage_ref ? [doc.storage_ref.path] : []));
359
+ const { documents, storagePaths: newStoragePaths, uploads } = await this.persistBucketData(bucket, [group.ops], context, undefined, { targetOp: group.targetOp });
360
+ const session = this.db.client.startSession();
361
+ try {
362
+ await session.withTransaction(async () => {
363
+ const verification = await bucketContext.collection
364
+ .aggregate([
365
+ { $match: { _id: { $in: idsToDelete } } },
366
+ {
367
+ $group: {
368
+ _id: null,
369
+ docCount: { $sum: 1 },
370
+ checksumSum: { $sum: '$checksum' },
371
+ opCountSum: { $sum: '$count' }
372
+ }
373
+ }
374
+ ], { session })
375
+ .next();
376
+ if (verification == null ||
377
+ verification.docCount !== expectedDocCount ||
378
+ verification.checksumSum !== expectedChecksum ||
379
+ verification.opCountSum !== expectedOpCount) {
380
+ throw new ConcurrentCompactionError(`Inputs changed while compacting bucket ${bucket}; restarting from the latest bucket state`);
381
+ }
382
+ await bucketContext.collection.deleteMany({ _id: { $in: idsToDelete } }, { session });
383
+ await bucketContext.collection.insertMany(documents, { session });
384
+ await this.finishObjectStorageReplacement(oldStoragePaths, newStoragePaths, uploads, session);
385
+ }, {
386
+ writeConcern: { w: 'majority' },
387
+ readConcern: { level: 'snapshot' }
59
388
  });
60
- if (bucketState == null) {
61
- return null;
389
+ }
390
+ finally {
391
+ await session.endSession();
392
+ }
393
+ return documents[0]._id;
394
+ }
395
+ /**
396
+ * Collapse the leading sequence of MOVE/REMOVE/CLEAR ops at the start
397
+ * of the bucket into a single CLEAR op. Reads whole clearable documents
398
+ * before the known boundary document, then splits that boundary document
399
+ * if it contains ops on both sides of lastNotPut.
400
+ *
401
+ * Returns the op count diff after replacing cleared ops with CLEAR ops.
402
+ */
403
+ async clearBucketLeading(lastNotPut, boundaryDocId, bucketContext, collection, context) {
404
+ let opCountDiff = 0;
405
+ const session = this.db.client.startSession();
406
+ try {
407
+ let done = false;
408
+ // First step is to clear full chunks that contain only CLEAR/MOVE/REMOVE operations.
409
+ // There can be many of them, so we do one batch at a time.
410
+ while (!done) {
411
+ const batch = await this.clearLeadingFullDocuments(session, lastNotPut, boundaryDocId, bucketContext, collection, context);
412
+ done = batch.done;
413
+ opCountDiff += batch.opCountDiff;
62
414
  }
63
- definitionId = bucketState._id.d;
415
+ // The final step is to process the "boundary" document: It may contain some CLEAR/MOVE/REMOVE operations,
416
+ // potentially followed by PUT operations. This is only a single document, so no need for batching.
417
+ opCountDiff += await this.clearBoundaryDocument(session, lastNotPut, boundaryDocId, bucketContext, collection, context);
64
418
  }
65
- return new SingleBucketStoreV3(this.db, { bucket, definitionId, replicationStreamId: this.group_id });
419
+ finally {
420
+ await session.endSession();
421
+ }
422
+ return opCountDiff;
423
+ }
424
+ async clearLeadingFullDocuments(session, lastNotPut, boundaryDocId, bucketContext, collection, context) {
425
+ const bucket = bucketContext.key.bucket;
426
+ this.signal?.throwIfAborted();
427
+ const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut]);
428
+ let done = false;
429
+ let opCountDiff = 0;
430
+ await session.withTransaction(async () => {
431
+ done = false;
432
+ opCountDiff = 0;
433
+ const oldStoragePaths = [];
434
+ const query = collection.find({
435
+ _id: {
436
+ $gte: bucketContext.minId,
437
+ $lt: boundaryDocId
438
+ }
439
+ }, {
440
+ session,
441
+ sort: { _id: 1 },
442
+ projection: {
443
+ _id: 1,
444
+ min_op: 1,
445
+ checksum: 1,
446
+ count: 1,
447
+ target_op: 1,
448
+ has_clear_op: 1,
449
+ storage_ref: 1
450
+ },
451
+ limit: this.clearBatchLimit
452
+ });
453
+ let combinedChecksum = 0;
454
+ let clearedOpCount = 0;
455
+ let maxTargetOp = null;
456
+ let lastDocId = null;
457
+ let clearOpCount = 0;
458
+ let gotNonClearOp = false;
459
+ for await (const doc of query.stream()) {
460
+ if (doc.min_op > lastNotPut) {
461
+ throw new ReplicationAssertionError(`Unexpected document before CLEAR boundary with min_op ${doc.min_op} > ${lastNotPut} in bucket ${bucket}`);
462
+ }
463
+ lastDocId = doc._id;
464
+ if (doc.storage_ref) {
465
+ oldStoragePaths.push(doc.storage_ref.path);
466
+ }
467
+ // The compaction scan established that every operation before the
468
+ // boundary is MOVE/REMOVE/CLEAR. Root metadata is sufficient to fold
469
+ // whole documents into one CLEAR, so avoid downloading their payloads.
470
+ if (doc.has_clear_op) {
471
+ clearOpCount++;
472
+ if (clearOpCount > 1) {
473
+ throw new ReplicationAssertionError(`Unexpected multiple CLEAR operations in bucket ${bucket}`);
474
+ }
475
+ }
476
+ if (!doc.has_clear_op || doc.count > 1) {
477
+ gotNonClearOp = true;
478
+ }
479
+ combinedChecksum = addChecksums(combinedChecksum, Number(doc.checksum));
480
+ clearedOpCount += doc.count;
481
+ if (doc.target_op != null && (maxTargetOp == null || doc.target_op > maxTargetOp)) {
482
+ maxTargetOp = doc.target_op;
483
+ }
484
+ }
485
+ if (!gotNonClearOp) {
486
+ done = true;
487
+ return;
488
+ }
489
+ this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastDocId?.o}`);
490
+ await collection.deleteMany({
491
+ _id: {
492
+ $gte: bucketContext.minId,
493
+ $lte: lastDocId
494
+ }
495
+ }, { session });
496
+ const clearOp = {
497
+ bucketKey: { ...context, bucket },
498
+ o: lastDocId.o,
499
+ op: 'CLEAR',
500
+ checksum: BigInt(combinedChecksum),
501
+ data: null
502
+ };
503
+ const persisted = await this.persistBucketData(bucket, [[clearOp]], context, prepared, {
504
+ targetOp: maxTargetOp
505
+ });
506
+ await collection.insertOne(persisted.documents[0], { session });
507
+ await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session);
508
+ opCountDiff = -clearedOpCount + 1;
509
+ }, {
510
+ writeConcern: { w: 'majority' },
511
+ readConcern: { level: 'snapshot' }
512
+ });
513
+ return { done, opCountDiff };
514
+ }
515
+ async clearBoundaryDocument(session, lastNotPut, boundaryDocId, bucketContext, collection, context) {
516
+ const bucket = bucketContext.key.bucket;
517
+ this.signal?.throwIfAborted();
518
+ const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut, boundaryDocId.o]);
519
+ let opCountDiff = 0;
520
+ await session.withTransaction(async () => {
521
+ opCountDiff = 0;
522
+ const oldStoragePaths = [];
523
+ const query = collection.find({
524
+ // This is a range query, but should only ever return two documents:
525
+ // 1. The CLEAR op from the previous clearLeadingFullDocuments.
526
+ // 2. The boundary document.
527
+ _id: {
528
+ $gte: bucketContext.minId,
529
+ $lte: boundaryDocId
530
+ }
531
+ }, {
532
+ session,
533
+ sort: { _id: 1 },
534
+ projection: {
535
+ _id: 1,
536
+ min_op: 1,
537
+ checksum: 1,
538
+ count: 1,
539
+ target_op: 1,
540
+ ops: 1,
541
+ storage_ref: 1
542
+ },
543
+ limit: 3
544
+ });
545
+ let docsRead = 0;
546
+ let combinedChecksum = 0;
547
+ let clearedOpCount = 0;
548
+ let maxTargetOp = null;
549
+ const boundarySurvivors = [];
550
+ for await (const doc of query.stream()) {
551
+ docsRead++;
552
+ if (docsRead > 2) {
553
+ throw new ReplicationAssertionError(`Unexpected extra document before CLEAR boundary in bucket ${bucket}`);
554
+ }
555
+ const isBoundaryDoc = doc._id.o == boundaryDocId.o;
556
+ if (doc.storage_ref) {
557
+ oldStoragePaths.push(doc.storage_ref.path);
558
+ }
559
+ await hydrateBucketDataDocuments([doc], this.storage.objectStorage, { signal: this.signal });
560
+ maxTargetOp = maxOpId(maxTargetOp, doc.target_op);
561
+ for (const op of loadBucketDataDocument(context, doc)) {
562
+ if (!isBoundaryDoc && op.op != 'CLEAR') {
563
+ throw new ReplicationAssertionError(`Unexpected ${op.op} operation before CLEAR boundary in bucket ${bucket}`);
564
+ }
565
+ if (op.o <= lastNotPut) {
566
+ if (op.op == 'PUT') {
567
+ throw new ReplicationAssertionError(`Unexpected PUT at op ${op.o} in CLEAR region for bucket ${bucket}`);
568
+ }
569
+ combinedChecksum = addChecksums(combinedChecksum, Number(op.checksum));
570
+ clearedOpCount++;
571
+ }
572
+ else if (isBoundaryDoc) {
573
+ boundarySurvivors.push(op);
574
+ }
575
+ else {
576
+ throw new ReplicationAssertionError(`Unexpected op ${op.o} after CLEAR boundary ${lastNotPut} in bucket ${bucket}`);
577
+ }
578
+ }
579
+ }
580
+ if (clearedOpCount == 0) {
581
+ throw new Error(`CLEAR boundary document not found for bucket ${bucket}`);
582
+ }
583
+ this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastNotPut}`);
584
+ await collection.deleteMany({
585
+ _id: {
586
+ $gte: bucketContext.minId,
587
+ $lte: boundaryDocId
588
+ }
589
+ }, { session });
590
+ const clearOp = {
591
+ bucketKey: { ...context, bucket },
592
+ o: lastNotPut,
593
+ op: 'CLEAR',
594
+ checksum: BigInt(combinedChecksum),
595
+ data: null
596
+ };
597
+ const chunks = [[clearOp]];
598
+ if (boundarySurvivors.length > 0) {
599
+ // These operations are a subset of one existing document, so keeping
600
+ // them together cannot increase its stored ops payload.
601
+ chunks.push(boundarySurvivors);
602
+ }
603
+ const persisted = await this.persistBucketData(bucket, chunks, context, prepared, {
604
+ targetOp: maxTargetOp ?? undefined
605
+ });
606
+ await collection.insertMany(persisted.documents, { session });
607
+ await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session);
608
+ opCountDiff = -clearedOpCount + 1;
609
+ }, {
610
+ writeConcern: { w: 'majority' },
611
+ readConcern: { level: 'snapshot' }
612
+ });
613
+ return opCountDiff;
614
+ }
615
+ /**
616
+ * Reserve stable object paths before starting a retryable MongoDB transaction.
617
+ * Each retry can safely overwrite the same paths, while the pre-existing
618
+ * deletion markers remain visible to the transaction that publishes them.
619
+ *
620
+ * CLEAR compaction reserves its maximum output count. Unused markers remain
621
+ * pending so they can clean up a path that an earlier transaction attempt may
622
+ * have uploaded before retrying with fewer output documents.
623
+ */
624
+ async prepareCompactionUploads(bucket, context, opIdHints) {
625
+ if (!this.storage.objectStorage) {
626
+ return [];
627
+ }
628
+ const lifecycle = this.objectStorageLifecycle;
629
+ const paths = opIdHints.map((opIdHint) => lifecycle.allocatePath(context.definitionId, bucket, opIdHint, opIdHint));
630
+ return lifecycle.prepareUploads(paths);
631
+ }
632
+ /** Publish replacement uploads and retire superseded objects in the same transaction. */
633
+ async finishObjectStorageReplacement(oldStoragePaths, newStoragePaths, uploads, session) {
634
+ if (!this.storage.objectStorage) {
635
+ return;
636
+ }
637
+ await this.objectStorageLifecycle.publishUploads(uploads, session);
638
+ await this.objectStorageLifecycle.retire(Array.from(oldStoragePaths).filter((path) => !newStoragePaths.has(path)), session);
639
+ }
640
+ async persistBucketData(bucket, chunks, context, preparedUploads, options) {
641
+ const serializedChunks = chunks.map((chunk) => serializeBucketData(bucket, chunk, options));
642
+ if (!this.storage.objectStorage) {
643
+ return {
644
+ documents: serializedChunks,
645
+ storagePaths: new Set(),
646
+ uploads: []
647
+ };
648
+ }
649
+ const store = new BucketDataObjectStorage(this.storage.objectStorage);
650
+ const storagePaths = new Set();
651
+ const lifecycle = this.objectStorageLifecycle;
652
+ // Base placement on the final compacted size. Unchanged documents are not
653
+ // rewritten, while small MOVE/merge results and CLEAR ops stay inline.
654
+ const storedIndexes = serializedChunks.flatMap((document, index) => document.size > this.storage.inlineThresholdBytes ? [index] : []);
655
+ const uploadsByIndex = new Map();
656
+ if (preparedUploads) {
657
+ for (const index of storedIndexes) {
658
+ const upload = preparedUploads[index];
659
+ if (!upload) {
660
+ throw new ServiceAssertionError(`Missing prepared object storage path for compacted document at index ${index}`);
661
+ }
662
+ uploadsByIndex.set(index, upload);
663
+ }
664
+ }
665
+ else {
666
+ const paths = storedIndexes.map((index) => {
667
+ const chunk = chunks[index];
668
+ return lifecycle.allocatePath(context.definitionId, bucket, chunk[0].o, chunk[chunk.length - 1].o);
669
+ });
670
+ const prepared = await lifecycle.prepareUploads(paths);
671
+ storedIndexes.forEach((index, preparedIndex) => uploadsByIndex.set(index, prepared[preparedIndex]));
672
+ }
673
+ // S3ObjectStorage applies one shared concurrency limit across all callers,
674
+ // so compaction can schedule its uploads together without creating a
675
+ // separate limiter here.
676
+ const documents = await Promise.all(serializedChunks.map(async (serialized, index) => {
677
+ const upload = uploadsByIndex.get(index);
678
+ if (!upload) {
679
+ return serialized;
680
+ }
681
+ const { ops, ...metadata } = serialized;
682
+ const { fileSize } = await store.store(upload.path, ops);
683
+ storagePaths.add(upload.path);
684
+ return {
685
+ ...metadata,
686
+ storage_ref: { path: upload.path, file_size: fileSize }
687
+ };
688
+ }));
689
+ return { documents, storagePaths, uploads: Array.from(uploadsByIndex.values()) };
66
690
  }
67
691
  }
68
692
  //# sourceMappingURL=MongoCompactorV3.js.map