@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -13,8 +13,10 @@ import {
13
13
  UpsertCurrentDataOptions
14
14
  } from '../common/PersistedBatch.js';
15
15
  import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
16
+ import { MongoWriteBatch } from '../MongoWriteBatch.js';
16
17
  import { serializeBucketData } from './bucket-format.js';
17
18
  import { chunkBucketData } from './chunking.js';
19
+ import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js';
18
20
  import {
19
21
  BucketDataDocumentV3,
20
22
  BucketStateDocumentV3,
@@ -24,12 +26,21 @@ import {
24
26
  taggedBucketParameterDocumentToTagged
25
27
  } from './models.js';
26
28
  import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
29
+ import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
27
30
  import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
28
31
 
32
+ interface SourceRecordWrite {
33
+ sourceTableId: bson.ObjectId;
34
+ operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3>;
35
+ }
36
+
29
37
  export class PersistedBatchV3 extends PersistedBatch {
30
- currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
38
+ // Upserts and soft deletes supply the complete source-record state, including
39
+ // pending_delete. Keep the final state per key, without removing any history.
40
+ currentData = new Map<string, SourceRecordWrite>();
31
41
  sourceTablePendingDeletes = new Map<string, InternalOpId>();
32
42
  protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
43
+ protected readonly objectStorageUsage?: ObjectStorageUsage;
33
44
 
34
45
  declare protected readonly db: VersionedPowerSyncMongoV3;
35
46
 
@@ -43,6 +54,7 @@ export class PersistedBatchV3 extends PersistedBatch {
43
54
  super(db, group_id, mapping, writtenSize, options);
44
55
  if (this.objectStorage) {
45
56
  this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
57
+ this.objectStorageUsage = new ObjectStorageUsage(this.db, this.group_id, this.objectStorageUsageWriterId);
46
58
  }
47
59
  }
48
60
 
@@ -127,7 +139,7 @@ export class PersistedBatchV3 extends PersistedBatch {
127
139
  }
128
140
 
129
141
  hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) {
130
- this.currentData.push({
142
+ this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
131
143
  sourceTableId,
132
144
  operation: {
133
145
  deleteOne: {
@@ -143,7 +155,7 @@ export class PersistedBatchV3 extends PersistedBatch {
143
155
  replicaId: storage.ReplicaId,
144
156
  checkpointGreaterThan: InternalOpId
145
157
  ) {
146
- this.currentData.push({
158
+ this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
147
159
  sourceTableId,
148
160
  operation: {
149
161
  updateOne: {
@@ -191,7 +203,7 @@ export class PersistedBatchV3 extends PersistedBatch {
191
203
  };
192
204
  });
193
205
 
194
- this.currentData.push({
206
+ this.currentData.set(this.currentDataKey(values.sourceTableId, values.replicaId), {
195
207
  sourceTableId: values.sourceTableId,
196
208
  operation: {
197
209
  updateOne: {
@@ -212,12 +224,12 @@ export class PersistedBatchV3 extends PersistedBatch {
212
224
  }
213
225
 
214
226
  protected get currentDataCount() {
215
- return this.currentData.length;
227
+ return this.currentData.size;
216
228
  }
217
229
 
218
230
  // Flush methods
219
231
 
220
- protected async flushBucketData(session: mongo.ClientSession) {
232
+ protected async queueBucketData(writes: MongoWriteBatch) {
221
233
  const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
222
234
  for (const document of this.bucketData) {
223
235
  const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
@@ -225,17 +237,17 @@ export class PersistedBatchV3 extends PersistedBatch {
225
237
  operationsByDefinition.set(document.bucketKey.definitionId, existing);
226
238
  }
227
239
 
228
- let uploadCount = 0;
229
240
  const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
230
241
  const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
231
- const lifecycle = this.objectStorage
232
- ? new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage)
233
- : undefined;
234
- const createInserts: (() => Promise<mongo.AnyBulkWriteOperation<BucketDataDocumentV3>>)[] = [];
242
+ const lifecycle = this.objectStorageLifecycle;
243
+ type BucketDataInsert = { insertOne: { document: BucketDataDocumentV3 } };
244
+ const createInserts: (() => Promise<BucketDataInsert>)[] = [];
235
245
 
236
246
  for (const [bucket, ops] of operationsByBucket) {
247
+ this.resetBucketPersistedBytes(definitionId, bucket);
237
248
  for (const chunk of chunkBucketData(ops)) {
238
249
  const serialized = serializeBucketData(bucket, chunk);
250
+ this.incrementBucketPersistedChunk(definitionId, bucket, serialized.size);
239
251
  if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
240
252
  createInserts.push(async () => ({
241
253
  insertOne: {
@@ -245,13 +257,12 @@ export class PersistedBatchV3 extends PersistedBatch {
245
257
  continue;
246
258
  }
247
259
 
248
- uploadCount += 1;
249
260
  createInserts.push(async () => {
250
261
  const minOp = chunk[0].o;
251
262
  const maxOp = chunk[chunk.length - 1].o;
252
263
  const { ops: bucketOps, ...metadata } = serialized;
253
264
  const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
254
- const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!);
265
+ const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!, { signal: this.signal });
255
266
  return {
256
267
  insertOne: {
257
268
  document: {
@@ -281,19 +292,29 @@ export class PersistedBatchV3 extends PersistedBatch {
281
292
  // S3ObjectStorage applies one shared concurrency limit across all callers,
282
293
  // so replication can schedule its uploads together without creating a
283
294
  // separate limiter here.
284
- const writes = await createAllInserts();
285
-
286
- for (const { definitionId, inserts } of writes) {
295
+ const bucketWrites = await createAllInserts();
296
+
297
+ const usageDeltas = this.objectStorageUsage == null ? undefined : new Map<BucketDefinitionId, bigint>();
298
+ for (const { definitionId, inserts } of bucketWrites) {
299
+ if (usageDeltas != null) {
300
+ const delta = inserts.reduce(
301
+ (sum, operation) => sum + ObjectStorageUsage.bytes(operation.insertOne.document),
302
+ 0n
303
+ );
304
+ if (delta !== 0n) {
305
+ usageDeltas.set(definitionId, delta);
306
+ }
307
+ }
287
308
  if (inserts.length > 0) {
288
- await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
289
- session,
290
- ordered: false
291
- });
309
+ writes.bulkWriteUnordered(this.db.bucketData(this.group_id, definitionId), inserts);
292
310
  }
293
311
  }
312
+ if (usageDeltas != null) {
313
+ this.objectStorageUsage!.applyDeltas(usageDeltas, writes);
314
+ }
294
315
  }
295
316
 
296
- protected async flushBucketParameters(session: mongo.ClientSession) {
317
+ protected queueBucketParameters(writes: MongoWriteBatch): void {
297
318
  const operationsByIndex = new Map<string, typeof this.bucketParameters>();
298
319
  for (const document of this.bucketParameters) {
299
320
  const existing = operationsByIndex.get(document.index) ?? [];
@@ -302,23 +323,20 @@ export class PersistedBatchV3 extends PersistedBatch {
302
323
  }
303
324
 
304
325
  for (const [indexId, documents] of operationsByIndex.entries()) {
305
- await this.db.parameterIndex(this.group_id, indexId).bulkWrite(
326
+ writes.bulkWriteUnordered(
327
+ this.db.parameterIndex(this.group_id, indexId),
306
328
  documents.map((document) => ({
307
329
  insertOne: {
308
330
  document: taggedBucketParameterDocumentToTagged(document)
309
331
  }
310
- })),
311
- {
312
- session,
313
- ordered: false
314
- }
332
+ }))
315
333
  );
316
334
  }
317
335
  }
318
336
 
319
- protected async flushCurrentData(session: mongo.ClientSession) {
320
- const operationsBySourceTable = new Map<string, typeof this.currentData>();
321
- for (const operation of this.currentData) {
337
+ protected queueCurrentData(writes: MongoWriteBatch): void {
338
+ const operationsBySourceTable = new Map<string, SourceRecordWrite[]>();
339
+ for (const operation of this.currentData.values()) {
322
340
  const sourceTableId = operation.sourceTableId.toHexString();
323
341
  const existing = operationsBySourceTable.get(sourceTableId) ?? [];
324
342
  existing.push(operation);
@@ -341,33 +359,31 @@ export class PersistedBatchV3 extends PersistedBatch {
341
359
  });
342
360
 
343
361
  if (sourceTableUpdates.length > 0) {
344
- await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
362
+ writes.bulkWriteUnordered(this.db.sourceTables(this.group_id), sourceTableUpdates);
345
363
  }
346
364
 
347
365
  for (const operations of operationsBySourceTable.values()) {
348
366
  const sourceTableId = operations[0]!.sourceTableId;
349
- await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(
350
- operations.map((entry) => entry.operation),
351
- {
352
- session,
353
- ordered: true
354
- }
367
+ writes.bulkWriteUnordered(
368
+ this.db.sourceRecords(this.group_id, sourceTableId),
369
+ operations.map((entry) => entry.operation)
355
370
  );
356
371
  }
357
372
  }
358
373
 
359
- protected async flushBucketStates(session: mongo.ClientSession) {
360
- await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
361
- session,
362
- ordered: false
363
- });
374
+ protected queueBucketStates(writes: MongoWriteBatch): void {
375
+ writes.bulkWriteUnordered(this.db.bucketState(this.group_id), this.getBucketStateUpdates());
364
376
  }
365
377
 
366
378
  protected resetCurrentData() {
367
- this.currentData = [];
379
+ this.currentData.clear();
368
380
  this.sourceTablePendingDeletes.clear();
369
381
  }
370
382
 
383
+ private currentDataKey(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): string {
384
+ return Buffer.from(bson.serialize({ t: sourceTableId, k: replicaId })).toString('base64');
385
+ }
386
+
371
387
  private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocumentV3>[] {
372
388
  return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => {
373
389
  if (state.definitionId == null) {
@@ -381,15 +397,42 @@ export class PersistedBatchV3 extends PersistedBatch {
381
397
  b: state.bucket
382
398
  }
383
399
  },
384
- update: {
385
- $set: {
386
- last_op: state.lastOp
387
- },
388
- $inc: {
389
- 'estimate_since_compact.count': state.incrementCount,
390
- 'estimate_since_compact.bytes': state.incrementBytes
400
+ // A pipeline update makes initialisation and scheduling one atomic
401
+ // writer operation. In particular, a later write cannot move an
402
+ // already-due compact check into the future.
403
+ update: [
404
+ {
405
+ $set: {
406
+ last_op: state.lastOp,
407
+ bucket_stats: {
408
+ count: { $add: [{ $ifNull: ['$bucket_stats.count', 0] }, state.incrementCount] },
409
+ bytes: { $add: [{ $ifNull: ['$bucket_stats.bytes', 0n] }, BigInt(state.incrementBytes)] },
410
+ chunks: { $add: [{ $ifNull: ['$bucket_stats.chunks', 0] }, state.incrementChunks] }
411
+ },
412
+ first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] },
413
+ next_compact_check: {
414
+ $let: {
415
+ vars: {
416
+ requested: {
417
+ $dateAdd: {
418
+ startDate: '$$NOW',
419
+ unit: 'millisecond',
420
+ amount: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS
421
+ }
422
+ }
423
+ },
424
+ in: {
425
+ $cond: [
426
+ { $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] },
427
+ '$next_compact_check',
428
+ '$$requested'
429
+ ]
430
+ }
431
+ }
432
+ }
433
+ }
391
434
  }
392
- },
435
+ ],
393
436
  upsert: true
394
437
  }
395
438
  } satisfies mongo.AnyBulkWriteOperation<BucketStateDocumentV3>;
@@ -1,5 +1,5 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
2
+ import { BucketDefinitionId, EventDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
3
3
  import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
4
4
  import { CommonSourceTableDocument } from '../models.js';
5
5
  import {
@@ -7,11 +7,19 @@ import {
7
7
  BucketParameterDocumentV3,
8
8
  BucketStateDocumentV3,
9
9
  CurrentDataDocumentV3,
10
+ CustomCheckpointRequestDocumentV3,
10
11
  ObjectStorageDeletionMarker,
12
+ ObjectStorageUsageDocument,
11
13
  SourceTableDocumentV3,
12
14
  SyncConfigDefinition
13
15
  } from './models.js';
14
16
 
17
+ export const OBJECT_STORAGE_USAGE_COLLECTION = 'object_storage_usage';
18
+ export type CustomCheckpointRequestCollectionSpecifier = {
19
+ replicationStreamId: number;
20
+ eventId: EventDefinitionId;
21
+ };
22
+
15
23
  export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
16
24
  constructor(
17
25
  upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
@@ -74,6 +82,42 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
74
82
  return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
75
83
  }
76
84
 
85
+ get objectStorageUsage(): mongo.Collection<ObjectStorageUsageDocument> {
86
+ return this.db.collection<ObjectStorageUsageDocument>(OBJECT_STORAGE_USAGE_COLLECTION);
87
+ }
88
+
89
+ customCheckpointRequests(options: CustomCheckpointRequestCollectionSpecifier) {
90
+ const collectionName = this.customCheckpointRequestCollectionName(options);
91
+ return this.db.collection<CustomCheckpointRequestDocumentV3>(collectionName);
92
+ }
93
+
94
+ async initializeCustomCheckpointRequestsCollection(options: CustomCheckpointRequestCollectionSpecifier) {
95
+ // createIndexes also creates the collection on first use. Callers invoke
96
+ // this lazily before the transaction which writes the first checkpoint.
97
+ await this.customCheckpointRequests(options).createIndexes([
98
+ {
99
+ key: { user_id: 1 },
100
+ name: 'user_unique',
101
+ unique: true
102
+ },
103
+ {
104
+ // Find checkpoint changes over an operation-id range.
105
+ key: { op_id: 1 },
106
+ name: 'op_id'
107
+ },
108
+ {
109
+ // Limit retention scans to temporary client-requested checkpoints.
110
+ key: { checkpoint_requested_at: 1 },
111
+ name: 'checkpoint_requested_at',
112
+ partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
113
+ }
114
+ ]);
115
+ }
116
+
117
+ listCustomCheckpointRequestCollections(replicationStreamId: number) {
118
+ return this.upstream.listCustomCheckpointRequestCollections(replicationStreamId);
119
+ }
120
+
77
121
  listBucketDataCollections(replicationStreamId: number) {
78
122
  return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
79
123
  }
@@ -117,11 +161,18 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
117
161
  },
118
162
  { name: 'bucket_updates', unique: true }
119
163
  );
164
+ // V3 workers claim only buckets that have reached their scheduled check
165
+ // time, keeping scheduled scans proportional to pending work.
120
166
  await bucketState.createIndex(
167
+ { next_compact_check: 1 },
121
168
  {
122
- 'estimate_since_compact.count': -1
123
- },
124
- { name: 'dirty_count' }
169
+ name: 'next_compact_check',
170
+ partialFilterExpression: { next_compact_check: { $exists: true } }
171
+ }
125
172
  );
126
173
  }
174
+
175
+ customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
176
+ return this.upstream.customCheckpointRequestCollectionName(options);
177
+ }
127
178
  }
@@ -32,6 +32,7 @@ export function serializeBucketData(
32
32
  op: op.op,
33
33
  source_table: op.source_table,
34
34
  source_key: op.source_key,
35
+ subkey: op.subkey,
35
36
  table: op.table,
36
37
  row_id: op.row_id,
37
38
  checksum: op.checksum,
@@ -78,6 +79,7 @@ export function* loadBucketDataDocument(
78
79
  op: op.op,
79
80
  source_table: op.source_table,
80
81
  source_key: op.source_key,
82
+ subkey: op.subkey,
81
83
  table: op.table,
82
84
  row_id: op.row_id,
83
85
  checksum: op.checksum,