@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,3 +1,5 @@
1
+ import * as timers from 'node:timers/promises';
2
+
1
3
  import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
2
4
  import {
3
5
  logger as defaultLogger,
@@ -5,24 +7,16 @@ import {
5
7
  ReplicationAssertionError,
6
8
  ServiceAssertionError
7
9
  } from '@powersync/lib-services-framework';
8
- import {
9
- addChecksums,
10
- InternalOpId,
11
- isPartialChecksum,
12
- PopulateChecksumCacheResults,
13
- storage,
14
- utils
15
- } from '@powersync/service-core';
10
+ import { InternalOpId, isPartialChecksum, PopulateChecksumCacheResults, storage } from '@powersync/service-core';
16
11
  import { BucketDefinitionId } from '@powersync/service-sync-rules';
17
12
 
18
- import { BucketDataDoc, BucketKey } from './common/BucketDataDoc.js';
19
- import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js';
13
+ import { BucketKey } from './common/BucketDataDoc.js';
20
14
  import type { VersionedPowerSyncMongo } from './db.js';
21
15
  import { BucketStateDocumentBase } from './models.js';
22
16
  import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
23
- import { cacheKey } from './OperationBatch.js';
17
+ import { isRetryableObjectStorageError } from './v3/object-storage/ObjectStorage.js';
24
18
 
25
- interface CurrentBucketState {
19
+ export interface CurrentBucketState {
26
20
  /** Bucket name */
27
21
  bucket: string;
28
22
  definitionId: BucketDefinitionId;
@@ -56,18 +50,26 @@ interface CurrentBucketState {
56
50
  opBytes: number;
57
51
  }
58
52
 
59
- type CompactClearProperties = 'op' | 'checksum' | 'target_op';
60
-
61
53
  export interface MongoCompactOptions extends storage.CompactOptions {}
62
54
 
63
55
  const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
64
56
  const DEFAULT_MOVE_BATCH_LIMIT = 2000;
65
57
  const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
58
+ const DEFAULT_MOVE_BATCH_BYTE_LIMIT = 16 * 1024 * 1024;
66
59
  const DEFAULT_MIN_BUCKET_CHANGES = 10;
67
60
  const DEFAULT_MIN_CHANGE_RATIO = 0.1;
68
61
  const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
69
62
  /** This default is primarily for tests. */
70
63
  const DEFAULT_MEMORY_LIMIT_MB = 64;
64
+ const COMPACTION_RETRY_LIMIT = 3;
65
+ const COMPACTION_RETRY_DELAY_MS = 1_000;
66
+
67
+ export class ConcurrentCompactionError extends Error {
68
+ constructor(message: string) {
69
+ super(message);
70
+ this.name = 'ConcurrentCompactionError';
71
+ }
72
+ }
71
73
 
72
74
  export interface DirtyBucket {
73
75
  bucket: string;
@@ -77,17 +79,18 @@ export interface DirtyBucket {
77
79
  }
78
80
 
79
81
  export abstract class MongoCompactor {
80
- protected updates: mongo.AnyBulkWriteOperation<BucketDataDocumentGeneric>[] = [];
81
82
  protected bucketStateUpdates: mongo.AnyBulkWriteOperation<BucketStateDocumentBase>[] = [];
82
83
 
83
84
  protected readonly idLimitBytes: number;
84
85
  protected readonly moveBatchLimit: number;
85
86
  protected readonly moveBatchQueryLimit: number;
87
+ protected readonly moveBatchByteLimit: number;
86
88
  protected readonly clearBatchLimit: number;
87
89
  protected readonly minBucketChanges: number;
88
90
  protected readonly minChangeRatio: number;
89
91
  protected readonly maxOpId: bigint;
90
92
  protected readonly buckets: string[] | undefined;
93
+ protected readonly deleteCheckpointRequestsBefore: Date | undefined;
91
94
  protected readonly signal?: AbortSignal;
92
95
  protected readonly group_id: number;
93
96
 
@@ -98,15 +101,20 @@ export abstract class MongoCompactor {
98
101
  protected readonly db: VersionedPowerSyncMongo,
99
102
  options: MongoCompactOptions
100
103
  ) {
101
- this.group_id = storage.group_id;
104
+ this.group_id = storage.replicationStreamId;
102
105
  this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
103
106
  this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
104
107
  this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
108
+ this.moveBatchByteLimit = options.moveBatchByteLimit ?? DEFAULT_MOVE_BATCH_BYTE_LIMIT;
105
109
  this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
110
+ if (this.clearBatchLimit < 2) {
111
+ throw new ReplicationAssertionError('clearBatchLimit must be >= 2');
112
+ }
106
113
  this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
107
114
  this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
108
115
  this.maxOpId = options.maxOpId ?? 0n;
109
116
  this.buckets = options.compactBuckets;
117
+ this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
110
118
  this.signal = options.signal;
111
119
  this.logger = options.logger ?? defaultLogger;
112
120
  }
@@ -114,9 +122,11 @@ export abstract class MongoCompactor {
114
122
  /**
115
123
  * Compact buckets by converting operations into MOVE and/or CLEAR operations.
116
124
  *
117
- * See /docs/compacting-operations.md for details.
125
+ * See /docs/storage/compacting-operations.md for details.
118
126
  */
119
127
  async compact() {
128
+ await this.deleteOldCheckpointRequests();
129
+
120
130
  if (this.buckets) {
121
131
  for (const bucket of this.buckets) {
122
132
  // We can make this more efficient later on by iterating through the buckets in a single query.
@@ -128,11 +138,31 @@ export abstract class MongoCompactor {
128
138
  }
129
139
  }
130
140
 
141
+ private async deleteOldCheckpointRequests() {
142
+ if (this.deleteCheckpointRequestsBefore == null) {
143
+ return;
144
+ }
145
+
146
+ this.signal?.throwIfAborted();
147
+ // The explicit $exists guarantees the query is a subset of the
148
+ // checkpoint_requested_at partial index's filter, so the planner can use it.
149
+ // Keep pending requests during replication lag so a retry doesn't advance the associated
150
+ // LSN even further.
151
+ await this.db.write_checkpoints.deleteMany({
152
+ checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore },
153
+ processed_at_lsn: { $ne: null }
154
+ });
155
+ await this.db.custom_write_checkpoints.deleteMany({
156
+ checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore }
157
+ });
158
+ }
159
+
131
160
  /**
132
161
  * Subset of compact, only populating checksums where relevant.
133
162
  */
134
163
  async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
135
164
  let count = 0;
165
+ // Paginate through dirty buckets in batches until no more buckets meet the criteria.
136
166
  while (true) {
137
167
  this.signal?.throwIfAborted();
138
168
  const buckets = await this.dirtyBucketBatchForChecksums(options);
@@ -172,6 +202,7 @@ export abstract class MongoCompactor {
172
202
  },
173
203
  getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null
174
204
  ): AsyncGenerator<DirtyBucket[]> {
205
+ // Paginate through the bucket state collection using cursor-based scanning.
175
206
  while (true) {
176
207
  // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline
177
208
  // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria.
@@ -300,207 +331,66 @@ export abstract class MongoCompactor {
300
331
  /**
301
332
  * Compaction for a single bucket, with retries on failure.
302
333
  *
303
- * This covers against occasional network or other database errors during a long compact job.
334
+ * A compaction can race another compactor after its initial scan. Restarting
335
+ * the bucket is safe because each replacement is transactional. Object
336
+ * storage paths are also prepared with lifecycle markers, so a retry can
337
+ * safely overwrite or eventually clean up uploads from the failed attempt.
304
338
  */
305
339
  protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) {
306
340
  let retryCount = 0;
307
341
  while (true) {
342
+ this.signal?.throwIfAborted();
343
+ // Do not carry queued bucket state writes from a failed attempt into the rescan.
344
+ this.bucketStateUpdates = [];
308
345
  try {
309
346
  await this.compactSingleBucket(bucket, definitionId);
310
- break;
347
+ return;
311
348
  } catch (e) {
312
- if (retryCount < 3 && isMongoServerError(e)) {
313
- this.logger.warn(`Error compacting bucket ${bucket}, retrying...`, e);
314
- retryCount++;
315
- await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount));
316
- } else {
349
+ if (this.signal?.aborted) {
317
350
  throw e;
318
351
  }
319
- }
320
- }
321
- }
322
-
323
- protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) {
324
- const idLimitBytes = this.idLimitBytes;
325
- const bucketContext = await this.getBucketDataContext(bucket, definitionId);
326
- if (bucketContext == null) {
327
- return;
328
- }
329
- const currentState: CurrentBucketState = {
330
- bucket,
331
- definitionId: bucketContext.key.definitionId,
332
- seen: new Map(),
333
- trackingSize: 0,
334
- lastNotPut: null,
335
- opsSincePut: 0,
336
- checksum: 0,
337
- opCount: 0,
338
- opBytes: 0
339
- };
340
352
 
341
- // Constant lower bound.
342
- const lowerBound = bucketContext.minId;
343
- // Upper bound is adjusted for each batch.
344
- let upperBound = bucketContext.maxId;
345
-
346
- while (true) {
347
- this.signal?.throwIfAborted();
348
-
349
- // Query one batch at a time, to avoid cursor timeouts.
350
- const pipeline = [
351
- {
352
- $match: {
353
- _id: {
354
- $gte: lowerBound,
355
- $lt: upperBound
356
- },
357
- // Workaround for a clustered collection bug where the $lt operator may include upperBound.
358
- // Technically only needed for storage V3.
359
- // https://jira.mongodb.org/browse/SERVER-121822
360
- '_id.o': { $lt: upperBound.o }
361
- }
362
- },
363
- { $sort: { _id: -1 } },
364
- { $limit: this.moveBatchQueryLimit },
365
- {
366
- $project: {
367
- _id: 1,
368
- op: 1,
369
- table: 1,
370
- row_id: 1,
371
- source_table: 1,
372
- source_key: 1,
373
- checksum: 1,
374
- size: { $bsonSize: '$$ROOT' }
375
- }
376
- }
377
- ];
378
-
379
- const cursor = bucketContext.collection.aggregate<BucketDataDocumentGeneric & { size: number | bigint }>(
380
- pipeline,
381
- {
382
- // batchSize is 1 more than limit to auto-close the cursor.
383
- // See https://github.com/mongodb/node-mongodb-native/pull/4580
384
- batchSize: this.moveBatchQueryLimit + 1
385
- }
386
- );
387
- // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
388
- // Instead, we load up to the limit.
389
- const rawBatch = await cursor.toArray();
390
- const batch = rawBatch.map((document) => {
391
- const { size, ...rest } = document;
392
- return {
393
- doc: bucketContext.fromPersistedDocument(rest),
394
- size
395
- };
396
- });
397
-
398
- if (batch.length == 0) {
399
- // We've reached the end.
400
- break;
401
- }
402
-
403
- // Reuse the exact collection _id value from Mongo for the next bound.
404
- upperBound = rawBatch[rawBatch.length - 1]._id;
405
-
406
- for (const { doc, size } of batch) {
407
- if (doc.o > this.maxOpId) {
408
- continue;
409
- }
410
-
411
- currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
412
- currentState.opCount += 1;
413
-
414
- let isPersistentPut = doc.op == 'PUT';
415
-
416
- currentState.opBytes += Number(size);
417
- if (doc.op == 'REMOVE' || doc.op == 'PUT') {
418
- const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`;
419
- const targetOp = currentState.seen.get(key);
420
- if (targetOp) {
421
- // Will convert to MOVE, so don't count as PUT.
422
- isPersistentPut = false;
423
-
424
- this.updates.push({
425
- updateOne: {
426
- filter: { _id: bucketContext.docId(doc.o) },
427
- update: {
428
- $set: {
429
- op: 'MOVE',
430
- target_op: targetOp
431
- },
432
- $unset: {
433
- source_table: 1,
434
- source_key: 1,
435
- table: 1,
436
- row_id: 1,
437
- data: 1
438
- }
439
- } satisfies mongo.UpdateFilter<BucketDataDocumentGeneric>
440
- }
441
- });
442
-
443
- // TODO: better estimate for this.
444
- currentState.opBytes += 200 - Number(size);
445
- } else if (currentState.trackingSize < idLimitBytes) {
446
- // flatstr reduces the memory usage by flattening the string.
447
- currentState.seen.set(utils.flatstr(key), doc.o);
448
- // length + 16 for the string
449
- // 24 for the bigint
450
- // 50 for map overhead
451
- // 50 for additional overhead
452
- currentState.trackingSize += key.length + 140;
453
- }
353
+ const retryReason = compactionRetryReason(e);
354
+ if (retryReason == null) {
355
+ throw e;
454
356
  }
455
-
456
- if (isPersistentPut) {
457
- currentState.lastNotPut = null;
458
- currentState.opsSincePut = 0;
459
- } else if (doc.op != 'CLEAR') {
460
- if (currentState.lastNotPut == null) {
461
- currentState.lastNotPut = doc.o;
462
- }
463
- currentState.opsSincePut += 1;
357
+ if (retryCount >= COMPACTION_RETRY_LIMIT) {
358
+ throw new Error(
359
+ `Failed to compact bucket ${bucket} after ${retryCount + 1} attempts (${retryReason}): ${errorMessage(e)}`,
360
+ { cause: e }
361
+ );
464
362
  }
465
363
 
466
- if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
467
- await this.flush(bucketContext);
468
- }
364
+ retryCount++;
365
+ const delay = COMPACTION_RETRY_DELAY_MS * 2 ** (retryCount - 1);
366
+ this.logger.warn(
367
+ `Error compacting bucket ${bucket} (${retryReason}); retrying in ${delay}ms ` +
368
+ `(attempt ${retryCount + 1}/${COMPACTION_RETRY_LIMIT + 1})`,
369
+ e
370
+ );
371
+ await timers.setTimeout(delay, undefined, this.signal ? { signal: this.signal } : undefined);
469
372
  }
470
-
471
- this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
472
- }
473
-
474
- // Free memory before clearing the bucket.
475
- currentState.seen.clear();
476
- if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
477
- this.logger.info(
478
- `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
479
- );
480
- // Need flush() before clear().
481
- await this.flush(bucketContext);
482
- await this.clearBucket(currentState, bucketContext);
483
373
  }
484
-
485
- // Do this after clearBucket so we have accurate counts.
486
- this.updateBucketChecksums(currentState);
487
- // Need another flush after updateBucketChecksums().
488
- await this.flush(bucketContext);
489
374
  }
490
375
 
491
- protected updateBucketChecksums(state: CurrentBucketState) {
376
+ protected abstract compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
377
+
378
+ protected collectBucketStateUpdates(
379
+ state: CurrentBucketState,
380
+ compactedOpId: InternalOpId
381
+ ): mongo.AnyBulkWriteOperation<BucketStateDocumentBase> {
492
382
  if (state.opCount < 0) {
493
383
  throw new ServiceAssertionError(
494
384
  `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`
495
385
  );
496
386
  }
497
- this.bucketStateUpdates.push({
387
+ return {
498
388
  updateOne: {
499
389
  filter: this.bucketStateFilter(state.bucket, state.definitionId),
500
390
  update: {
501
391
  $set: {
502
392
  compacted_state: {
503
- op_id: this.maxOpId,
393
+ op_id: compactedOpId,
504
394
  count: state.opCount,
505
395
  checksum: BigInt(state.checksum),
506
396
  bytes: state.opBytes
@@ -517,24 +407,14 @@ export abstract class MongoCompactor {
517
407
  // We don't create new ones here, to avoid issues with the unique index on bucket_updates.
518
408
  upsert: false
519
409
  }
520
- });
410
+ };
521
411
  }
522
412
 
523
- protected async flush(col: SingleBucketStore) {
524
- if (this.updates.length > 0) {
525
- this.logger.info(`Compacting ${this.updates.length} ops`);
526
- await col.collection.bulkWrite(this.updates, {
527
- // Order is not important. Since checksums are not affected, these operations can happen in any order,
528
- // and it's fine if the operations are partially applied. Each individual operation is atomic.
529
- ordered: false
530
- });
531
- this.updates = [];
532
- }
533
-
534
- await this.flushBucketStateUpdates();
413
+ protected updateBucketChecksums(state: CurrentBucketState, compactedOpId: InternalOpId) {
414
+ this.bucketStateUpdates.push(this.collectBucketStateUpdates(state, compactedOpId));
535
415
  }
536
416
 
537
- private async flushBucketStateUpdates() {
417
+ protected async flushBucketStateUpdates() {
538
418
  if (this.bucketStateUpdates.length > 0) {
539
419
  this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
540
420
  await this.writeBucketStateUpdates();
@@ -542,108 +422,6 @@ export abstract class MongoCompactor {
542
422
  }
543
423
  }
544
424
 
545
- /**
546
- * Perform a CLEAR compact for a bucket.
547
- *
548
- * @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
549
- */
550
- protected async clearBucket(currentState: CurrentBucketState, col: SingleBucketStore) {
551
- const clearOp = currentState.lastNotPut!;
552
-
553
- const opFilter = {
554
- _id: {
555
- $gte: col.minId,
556
- $lte: col.docId(clearOp)
557
- }
558
- };
559
-
560
- const session = this.db.client.startSession();
561
- try {
562
- let done = false;
563
- while (!done) {
564
- this.signal?.throwIfAborted();
565
- let opCountDiff = 0;
566
- // Do the CLEAR operation in batches, with each batch a separate transaction.
567
- // The state after each batch is fully consistent.
568
- // We need a transaction per batch to make sure checksums stay consistent.
569
- await session.withTransaction(
570
- async () => {
571
- const query = col.collection.find<Pick<BucketDataDocumentGeneric, '_id' | CompactClearProperties>>(
572
- opFilter,
573
- {
574
- session,
575
- sort: { _id: 1 },
576
- projection: {
577
- _id: 1,
578
- op: 1,
579
- checksum: 1,
580
- target_op: 1
581
- },
582
- limit: this.clearBatchLimit
583
- }
584
- );
585
- let checksum = 0;
586
- let lastOp: Pick<BucketDataDoc, 'o' | CompactClearProperties> | null = null;
587
- let targetOp: bigint | null = null;
588
- let gotAnOp = false;
589
- let numberOfOpsToClear = 0;
590
- for await (const rawOp of query.stream()) {
591
- const op = col.fromPartialPersistedDocument(rawOp);
592
-
593
- if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
594
- checksum = utils.addChecksums(checksum, Number(op.checksum));
595
- lastOp = op;
596
- numberOfOpsToClear += 1;
597
- if (op.op != 'CLEAR') {
598
- gotAnOp = true;
599
- }
600
- if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
601
- targetOp = op.target_op;
602
- }
603
- } else {
604
- throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
605
- }
606
- }
607
- if (!gotAnOp) {
608
- done = true;
609
- return;
610
- }
611
-
612
- this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
613
- await col.collection.deleteMany(
614
- {
615
- _id: {
616
- $gte: col.minId,
617
- $lte: col.docId(lastOp!.o)
618
- }
619
- },
620
- { session }
621
- );
622
-
623
- const op = col.toPersistedDocument({
624
- o: lastOp!.o,
625
- op: 'CLEAR',
626
- checksum: BigInt(checksum),
627
- data: null,
628
- target_op: targetOp
629
- });
630
- await col.collection.insertOne(op, { session });
631
-
632
- opCountDiff = -numberOfOpsToClear + 1;
633
- },
634
- {
635
- writeConcern: { w: 'majority' },
636
- readConcern: { level: 'snapshot' }
637
- }
638
- );
639
- // Update outside the transaction, since the transaction can be retried multiple times.
640
- currentState.opCount += opCountDiff;
641
- }
642
- } finally {
643
- await session.endSession();
644
- }
645
- }
646
-
647
425
  protected async updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]) {
648
426
  const checksums = await this.computeChecksumsForBuckets(buckets);
649
427
  const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
@@ -684,23 +462,31 @@ export abstract class MongoCompactor {
684
462
  await this.flushBucketStateUpdates();
685
463
  }
686
464
 
687
- protected formatBucketDataKey(doc: Pick<BucketDataDoc, 'bucketKey' | 'o'>) {
688
- return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
689
- }
690
-
691
465
  protected abstract writeBucketStateUpdates(): Promise<void>;
692
466
  protected abstract computeChecksumsForBuckets(
693
467
  buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]
694
468
  ): Promise<storage.PartialChecksumMap>;
695
469
  protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document;
696
-
697
- protected abstract getBucketDataContext(
698
- bucket: string,
699
- definitionId: BucketDefinitionId | null
700
- ): Promise<SingleBucketStore | null>;
701
470
  }
702
471
 
703
472
  export interface BucketDataCollectionContext<TBucketData extends mongo.Document> {
704
473
  bucketKey: BucketKey;
705
474
  collection: mongo.Collection<TBucketData>;
706
475
  }
476
+
477
+ function compactionRetryReason(error: unknown): string | null {
478
+ if (error instanceof ConcurrentCompactionError) {
479
+ return 'concurrent compaction';
480
+ }
481
+ if (isRetryableObjectStorageError(error)) {
482
+ return 'transient object storage failure';
483
+ }
484
+ if (isMongoServerError(error)) {
485
+ return 'MongoDB failure';
486
+ }
487
+ return null;
488
+ }
489
+
490
+ function errorMessage(error: unknown): string {
491
+ return error instanceof Error ? error.message : String(error);
492
+ }
@@ -0,0 +1,61 @@
1
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
2
+ import { SyncConfigWithRequiredMapping } from '@powersync/service-core';
3
+ import {
4
+ BucketDataScope,
5
+ BucketDataSource,
6
+ HydrationState,
7
+ ParameterIndexLookupCreator,
8
+ ParameterLookupScope
9
+ } from '@powersync/service-sync-rules';
10
+
11
+ export class MongoHydrationState implements HydrationState {
12
+ private bucketDataSourceSyncConfig = new WeakMap<BucketDataSource, SyncConfigWithRequiredMapping>();
13
+ private parameterIndexLookupSyncConfig = new WeakMap<ParameterIndexLookupCreator, SyncConfigWithRequiredMapping>();
14
+
15
+ constructor(
16
+ readonly syncConfigs: SyncConfigWithRequiredMapping[],
17
+ private readonly replicationStreamId: number
18
+ ) {
19
+ for (let syncConfig of syncConfigs) {
20
+ for (let source of syncConfig.syncConfig.config.bucketDataSources) {
21
+ this.bucketDataSourceSyncConfig.set(source, syncConfig);
22
+ }
23
+ for (let source of syncConfig.syncConfig.config.bucketParameterLookupSources) {
24
+ this.parameterIndexLookupSyncConfig.set(source, syncConfig);
25
+ }
26
+ }
27
+ }
28
+
29
+ getBucketSourceScope(source: BucketDataSource): BucketDataScope {
30
+ const syncConfig = this.bucketDataSourceSyncConfig.get(source);
31
+ if (syncConfig == null) {
32
+ throw new ServiceAssertionError(`No sync config found for bucket data source ${source.uniqueName}`);
33
+ }
34
+ const mapping = syncConfig.mapping;
35
+ const defId = mapping.bucketSourceId(source);
36
+ // Currently uniqueName is constant for a BucketDataSource within a replication stream. This may change in the
37
+ // future, in which case we need to reconsider the bucketPrefix here.
38
+ // However, the definition may change without changing the name, so we include the defId in the bucketPrefix.
39
+ // defId is not unique across replication streams, so we include the replicationStreamId as well.
40
+ return {
41
+ bucketPrefix: `${source.uniqueName}.${this.replicationStreamId.toString(16)}.${defId}`,
42
+ source
43
+ };
44
+ }
45
+
46
+ getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope {
47
+ const syncConfig = this.parameterIndexLookupSyncConfig.get(source);
48
+ if (syncConfig == null) {
49
+ throw new ServiceAssertionError(
50
+ `No sync config found for parameter index lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId}`
51
+ );
52
+ }
53
+ const mapping = syncConfig.mapping;
54
+ const defId = mapping.parameterLookupId(source);
55
+ return {
56
+ lookupName: defId,
57
+ queryId: '',
58
+ source
59
+ };
60
+ }
61
+ }