@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/CHANGELOG.md +62 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +36 -3
  3. package/dist/storage/MongoBucketStorage.js +372 -207
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +17 -5
  6. package/dist/storage/implementation/MongoBucketBatch.js +44 -17
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +19 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  12. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  13. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  14. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  15. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +29 -0
  16. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  17. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +4 -2
  18. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  19. package/dist/storage/implementation/MongoStorageProvider.js +7 -3
  20. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  21. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +27 -11
  22. package/dist/storage/implementation/MongoSyncBucketStorage.js +78 -31
  23. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  24. package/dist/storage/implementation/MongoSyncRulesLock.js +21 -9
  25. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  26. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  27. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  28. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  29. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  30. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  31. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  32. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  33. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  34. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  35. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  36. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  37. package/dist/storage/implementation/common/PersistedBatch.js +49 -6
  38. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  39. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  40. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  41. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  42. package/dist/storage/implementation/db.d.ts +9 -1
  43. package/dist/storage/implementation/db.js +27 -1
  44. package/dist/storage/implementation/db.js.map +1 -1
  45. package/dist/storage/implementation/models.d.ts +31 -0
  46. package/dist/storage/implementation/models.js.map +1 -1
  47. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +2 -0
  48. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +33 -4
  49. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  50. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  51. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  52. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  53. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  54. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  55. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  56. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  57. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +65 -16
  58. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  59. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  60. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  61. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  62. package/dist/storage/implementation/v1/models.js +1 -0
  63. package/dist/storage/implementation/v1/models.js.map +1 -1
  64. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  65. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  66. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  67. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +10 -2
  68. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +100 -26
  69. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  70. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +82 -25
  71. package/dist/storage/implementation/v3/MongoCompactorV3.js +698 -159
  72. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  73. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  74. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  75. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  76. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  77. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  78. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  79. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  80. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +91 -31
  81. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  82. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  83. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  84. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  85. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  86. package/dist/storage/implementation/v3/PersistedBatchV3.js +76 -42
  87. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  88. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  89. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  90. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  91. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  92. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  93. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  94. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  95. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  96. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  97. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  98. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  99. package/dist/storage/implementation/v3/models.d.ts +84 -4
  100. package/dist/storage/implementation/v3/models.js.map +1 -1
  101. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  102. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  103. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  104. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  105. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  106. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  107. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  108. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  109. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  110. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  111. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  112. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  113. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  114. package/dist/storage/implementation/v3/source-table-utils.d.ts +6 -6
  115. package/dist/storage/implementation/v3/source-table-utils.js +30 -37
  116. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  117. package/dist/storage/storage-index.d.ts +2 -1
  118. package/dist/storage/storage-index.js +1 -0
  119. package/dist/storage/storage-index.js.map +1 -1
  120. package/dist/types/types.d.ts +28 -1
  121. package/dist/types/types.js +28 -7
  122. package/dist/types/types.js.map +1 -1
  123. package/dist/utils/test-utils.js +1 -0
  124. package/dist/utils/test-utils.js.map +1 -1
  125. package/dist/utils/util.js +3 -1
  126. package/dist/utils/util.js.map +1 -1
  127. package/package.json +10 -9
  128. package/src/storage/MongoBucketStorage.ts +507 -269
  129. package/src/storage/implementation/MongoBucketBatch.ts +67 -24
  130. package/src/storage/implementation/MongoCompactor.ts +35 -346
  131. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  132. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +34 -0
  133. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +4 -2
  134. package/src/storage/implementation/MongoStorageProvider.ts +15 -3
  135. package/src/storage/implementation/MongoSyncBucketStorage.ts +114 -40
  136. package/src/storage/implementation/MongoSyncRulesLock.ts +22 -9
  137. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  138. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  139. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  140. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  141. package/src/storage/implementation/common/PersistedBatch.ts +66 -10
  142. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  143. package/src/storage/implementation/db.ts +35 -2
  144. package/src/storage/implementation/models.ts +34 -0
  145. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +46 -6
  146. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  147. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  148. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +87 -27
  149. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  150. package/src/storage/implementation/v1/models.ts +1 -0
  151. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  152. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +138 -34
  153. package/src/storage/implementation/v3/MongoCompactorV3.ts +958 -225
  154. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  155. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  156. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +111 -41
  157. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  158. package/src/storage/implementation/v3/PersistedBatchV3.ts +93 -48
  159. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  160. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  161. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  162. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  163. package/src/storage/implementation/v3/models.ts +91 -3
  164. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  165. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  166. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  167. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  168. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  169. package/src/storage/implementation/v3/source-table-utils.ts +39 -41
  170. package/src/storage/storage-index.ts +2 -0
  171. package/src/types/types.ts +42 -7
  172. package/src/utils/test-utils.ts +1 -0
  173. package/src/utils/util.ts +3 -1
  174. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  175. package/test/src/__snapshots__/storage_sync.test.ts.snap +605 -16
  176. package/test/src/chunk_compaction_config.test.ts +38 -0
  177. package/test/src/cleanup-stopped-sync-configs.test.ts +40 -13
  178. package/test/src/compact-utils.test.ts +226 -0
  179. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  180. package/test/src/object_storage_usage.test.ts +267 -0
  181. package/test/src/parameter_compacting_v1.test.ts +120 -0
  182. package/test/src/parameter_compacting_v3.test.ts +192 -0
  183. package/test/src/parameter_compaction_fence.test.ts +276 -0
  184. package/test/src/storage.test.ts +273 -22
  185. package/test/src/storage_compacting.test.ts +968 -246
  186. package/test/src/storage_s3_checksums.test.ts +3 -4
  187. package/test/src/storage_s3_compaction_lifecycle.test.ts +138 -12
  188. package/test/src/storage_s3_reading.test.ts +290 -2
  189. package/test/src/storage_s3_writing.test.ts +1 -0
  190. package/test/src/storage_sync.test.ts +336 -41
  191. package/test/src/util.ts +1 -1
  192. package/tsconfig.tsbuildinfo +1 -1
@@ -1,29 +1,31 @@
1
- import { mongo } from '@powersync/lib-service-mongodb';
2
1
  import * as framework from '@powersync/lib-services-framework';
3
- import { GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
2
+ import { GetCheckpointChangesOptions, storage } from '@powersync/service-core';
4
3
  import { VersionedPowerSyncMongo } from './db.js';
5
4
 
6
5
  export type MongoCheckpointAPIOptions = {
7
6
  db: VersionedPowerSyncMongo;
8
- mode: storage.WriteCheckpointMode;
9
- sync_rules_id: number;
7
+ writeCheckpointMode: storage.WriteCheckpointModeConfig;
8
+ replicationStreamId: number;
10
9
  };
11
10
 
12
11
  export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
13
12
  readonly db: VersionedPowerSyncMongo;
13
+ readonly replicationStreamId: number;
14
+
14
15
  private _mode: storage.WriteCheckpointMode;
15
16
 
16
17
  constructor(options: MongoCheckpointAPIOptions) {
17
18
  this.db = options.db;
18
- this._mode = options.mode;
19
+ this._mode = options.writeCheckpointMode.mode;
20
+ this.replicationStreamId = options.replicationStreamId;
19
21
  }
20
22
 
21
23
  get writeCheckpointMode() {
22
24
  return this._mode;
23
25
  }
24
26
 
25
- setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
26
- this._mode = mode;
27
+ setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
28
+ this._mode = config.mode;
27
29
  }
28
30
 
29
31
  async createManagedWriteCheckpoints(
@@ -93,7 +95,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
93
95
  },
94
96
  upsert: true
95
97
  }
96
- }))
98
+ })),
99
+ { ordered: false }
97
100
  );
98
101
 
99
102
  const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
@@ -178,7 +181,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
178
181
  upsert: true
179
182
  }
180
183
  };
181
- })
184
+ }),
185
+ { ordered: false }
182
186
  );
183
187
 
184
188
  // Fetch the final ids separately so stale requests can still return the
@@ -247,6 +251,7 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
247
251
  }
248
252
 
249
253
  protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) {
254
+ // Legacy V1/V2 implementation.
250
255
  const { user_id, sync_rules_id } = filters;
251
256
  const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
252
257
  user_id,
@@ -301,8 +306,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
301
306
  };
302
307
  }
303
308
 
304
- private async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
309
+ protected async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
305
310
  const limit = 1000;
311
+ // Legacy V1/V2 implementation.
306
312
  const changes = await this.db.custom_write_checkpoints
307
313
  .find(
308
314
  {
@@ -332,39 +338,3 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
332
338
  };
333
339
  }
334
340
  }
335
-
336
- export async function batchCreateCustomWriteCheckpoints(
337
- db: VersionedPowerSyncMongo,
338
- session: mongo.ClientSession,
339
- checkpoints: storage.CustomWriteCheckpointOptions[],
340
- opId: InternalOpId
341
- ): Promise<void> {
342
- if (checkpoints.length == 0) {
343
- return;
344
- }
345
-
346
- await db.custom_write_checkpoints.bulkWrite(
347
- checkpoints.map((checkpointOptions) => {
348
- const set: Record<string, unknown> = {
349
- checkpoint: checkpointOptions.checkpoint,
350
- sync_rules_id: checkpointOptions.sync_rules_id,
351
- op_id: opId
352
- };
353
- if (checkpointOptions.checkpoint_requested_at != null) {
354
- set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
355
- }
356
-
357
- return {
358
- updateOne: {
359
- filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
360
- update: {
361
- $set: set,
362
- ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
363
- },
364
- upsert: true
365
- }
366
- };
367
- }),
368
- { session }
369
- );
370
- }
@@ -2,21 +2,28 @@ import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { storage } from '@powersync/service-core';
3
3
 
4
4
  /**
5
- * Update pipeline to update replication stream status, covering all storage versions.
5
+ * Update pipeline to stop a deploying replication stream, covering all storage versions.
6
6
  *
7
7
  * Roughly equivalent to:
8
8
  * $set: {
9
- * state: state,
10
- * 'sync_configs.$[].state': state
9
+ * state: STOP,
10
+ * 'sync_configs.$[].state': STOP
11
11
  * }
12
12
  *
13
13
  * The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
14
+ *
15
+ * This can be used for specific use cases:
16
+ * 1. Stop an existing PROCESSING stream, when it is being replaced by a new PROCESSING one.
17
+ * 2. Stop an existing ACTIVE or ERRORED stream, when it is being replaced by a new ACTIVE one.
18
+ *
19
+ * Do not use this when an ACTIVE stream transitions to ERRORED, and is being replaced by a new PROCESSING one.
20
+ * In that case, we need to maintain the ERRORED state, which this pipeline does not cover.
14
21
  */
15
- export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo.Document[] {
22
+ export function stopReplicationStreamPipeline(): mongo.Document[] {
16
23
  return [
17
24
  {
18
25
  $set: {
19
- state,
26
+ state: storage.SyncRuleState.STOP,
20
27
  sync_configs: {
21
28
  $cond: [
22
29
  { $isArray: '$sync_configs' },
@@ -25,7 +32,7 @@ export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo
25
32
  input: '$sync_configs',
26
33
  as: 'config',
27
34
  in: {
28
- $mergeObjects: ['$$config', { state }]
35
+ $mergeObjects: ['$$config', { state: storage.SyncRuleState.STOP }]
29
36
  }
30
37
  }
31
38
  },
@@ -1,9 +1,31 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { InternalOpId } from '@powersync/service-core';
2
+ import { InternalOpId, ReplicationCheckpoint } from '@powersync/service-core';
3
3
  import * as bson from 'bson';
4
4
 
5
- export interface MongoSyncBucketStorageCheckpoint {
5
+ export interface MongoSyncBucketStorageCheckpoint extends ReplicationCheckpoint {
6
6
  checkpoint: InternalOpId;
7
7
  snapshotTime: bson.Timestamp;
8
8
  clusterTime: mongo.ClusterTime;
9
+
10
+ /**
11
+ * The stream's `parameter_compaction.checkpoint_changes_invalid_before` boundary, read in the
12
+ * same snapshot as {@link checkpoint} and {@link snapshotTime}.
13
+ *
14
+ * Parameter compaction may have deleted parameter entries below this boundary, so checkpoint
15
+ * change detection cannot enumerate individual lookup changes from a checkpoint below it.
16
+ *
17
+ * 0n for streams that have never been compacted.
18
+ */
19
+ parameterChangesInvalidBefore: InternalOpId;
20
+ }
21
+
22
+ /**
23
+ * MongoDB-specific version of GetCheckpointChangesOptions.
24
+ *
25
+ * The next checkpoint carries the snapshot and invalidation boundary that change detection must
26
+ * be evaluated against. Only `checkpoint` is used from the previous one.
27
+ */
28
+ export interface MongoGetCheckpointChangesOptions {
29
+ lastCheckpoint: ReplicationCheckpoint;
30
+ nextCheckpoint: MongoSyncBucketStorageCheckpoint;
9
31
  }
@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
8
8
  import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
9
9
  import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
10
10
  import { MongoIdSequence } from '../MongoIdSequence.js';
11
+ import { MongoWriteBatch } from '../MongoWriteBatch.js';
11
12
  import type { VersionedPowerSyncMongo } from '../db.js';
12
13
  import { TaggedBucketParameterDocument } from '../models.js';
13
14
  import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
@@ -65,6 +66,11 @@ export interface PersistedBatchOptions {
65
66
  logger?: Logger;
66
67
  objectStorage?: ObjectStorage;
67
68
  inlineThresholdBytes?: number;
69
+ /**
70
+ * Aborts in-flight object storage uploads when replication stops.
71
+ */
72
+ signal?: AbortSignal;
73
+ objectStorageUsageWriterId?: string;
68
74
  }
69
75
 
70
76
  /**
@@ -81,6 +87,8 @@ export abstract class PersistedBatch {
81
87
 
82
88
  protected readonly objectStorage?: ObjectStorage;
83
89
  protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
90
+ protected readonly signal?: AbortSignal;
91
+ protected readonly objectStorageUsageWriterId?: string;
84
92
 
85
93
  /**
86
94
  * For debug logging only.
@@ -102,6 +110,8 @@ export abstract class PersistedBatch {
102
110
  this.currentSize = writtenSize;
103
111
  this.logger = options?.logger ?? defaultLogger;
104
112
  this.objectStorage = options?.objectStorage;
113
+ this.signal = options?.signal;
114
+ this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
105
115
  if (options?.inlineThresholdBytes != null) {
106
116
  this.inlineThresholdBytes = options.inlineThresholdBytes;
107
117
  }
@@ -124,7 +134,8 @@ export abstract class PersistedBatch {
124
134
  remaining_buckets.set(currentBucketKey(mapped), mapped);
125
135
  }
126
136
 
127
- const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)));
137
+ const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
138
+ const dchecksum = BigInt(utils.hashDelete(subkey));
128
139
 
129
140
  for (const evaluated of options.evaluated) {
130
141
  const definitionId = this.getBucketDefinitionId(evaluated.source);
@@ -159,6 +170,7 @@ export abstract class PersistedBatch {
159
170
  bucket: evaluated.bucket,
160
171
  sourceTableId: options.table.id,
161
172
  sourceKey: options.sourceKey,
173
+ subkey,
162
174
  table: evaluated.table,
163
175
  rowId: evaluated.id,
164
176
  checksum: BigInt(checksum),
@@ -181,6 +193,7 @@ export abstract class PersistedBatch {
181
193
  op_id,
182
194
  sourceTableId: options.table.id,
183
195
  sourceKey: options.sourceKey,
196
+ subkey,
184
197
  table: bucket.table,
185
198
  rowId: bucket.id,
186
199
  checksum: dchecksum
@@ -204,19 +217,27 @@ export abstract class PersistedBatch {
204
217
 
205
218
  protected abstract get currentDataCount(): number;
206
219
 
207
- protected abstract flushBucketData(session: mongo.ClientSession): Promise<void>;
220
+ protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
208
221
 
209
- protected abstract flushBucketParameters(session: mongo.ClientSession): Promise<void>;
222
+ protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
210
223
 
211
- protected abstract flushCurrentData(session: mongo.ClientSession): Promise<void>;
224
+ protected abstract queueCurrentData(writes: MongoWriteBatch): void;
212
225
 
213
- protected abstract flushBucketStates(session: mongo.ClientSession): Promise<void>;
226
+ protected abstract queueBucketStates(writes: MongoWriteBatch): void;
214
227
 
215
228
  protected abstract resetCurrentData(): void;
216
229
 
217
230
  protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
218
231
  protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
219
232
 
233
+ /**
234
+ * New bucket operations persist protocol subkeys during replication. Older
235
+ * V1 documents may not have one, so sync reads retain a fallback.
236
+ */
237
+ protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
238
+ return replicaIdToSubkey(table, key);
239
+ }
240
+
220
241
  protected get bucketDataCount(): number {
221
242
  return this.bucketData.length;
222
243
  }
@@ -234,17 +255,45 @@ export abstract class PersistedBatch {
234
255
  bucket,
235
256
  lastOp: op_id,
236
257
  incrementCount: 1,
237
- incrementBytes: bytes
258
+ incrementBytes: bytes,
259
+ incrementChunks: 0
238
260
  });
239
261
  }
240
262
  }
241
263
 
264
+ /**
265
+ * V3 persists operations in chunks. Keep this separate from incrementBucket:
266
+ * operation counts are known while evaluating rows, while chunk counts and
267
+ * exact persisted bytes are only known after the writer has chunked a flush.
268
+ */
269
+ protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
270
+ const key = `${definitionId ?? ''}:${bucket}`;
271
+ const existingState = this.bucketStates.get(key);
272
+ if (existingState != null) {
273
+ existingState.incrementChunks += 1;
274
+ existingState.incrementBytes += bytes;
275
+ }
276
+ }
277
+
278
+ /**
279
+ * V3's compactor calculates byte deltas from persisted chunk metadata.
280
+ * Replace the writer's per-operation estimate with those exact sizes before
281
+ * flushing the corresponding bucket state.
282
+ */
283
+ protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
284
+ const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
285
+ if (state != null) {
286
+ state.incrementBytes = 0;
287
+ }
288
+ }
289
+
242
290
  protected addBucketDataPut(options: {
243
291
  op_id: InternalOpId;
244
292
  bucketKey: BucketKey;
245
293
  bucket: string;
246
294
  sourceTableId: storage.SourceTable['id'];
247
295
  sourceKey: storage.ReplicaId;
296
+ subkey: string;
248
297
  table: string;
249
298
  rowId: string;
250
299
  checksum: bigint;
@@ -256,6 +305,7 @@ export abstract class PersistedBatch {
256
305
  op: 'PUT',
257
306
  source_table: mongoTableId(options.sourceTableId),
258
307
  source_key: options.sourceKey,
308
+ subkey: options.subkey,
259
309
  table: options.table,
260
310
  row_id: options.rowId,
261
311
  checksum: options.checksum,
@@ -268,6 +318,7 @@ export abstract class PersistedBatch {
268
318
  bucketKey: BucketKey;
269
319
  sourceTableId: storage.SourceTable['id'];
270
320
  sourceKey: storage.ReplicaId;
321
+ subkey: string;
271
322
  table: string;
272
323
  rowId: string;
273
324
  checksum: bigint;
@@ -278,6 +329,7 @@ export abstract class PersistedBatch {
278
329
  op: 'REMOVE',
279
330
  source_table: mongoTableId(options.sourceTableId),
280
331
  source_key: options.sourceKey,
332
+ subkey: options.subkey,
281
333
  table: options.table,
282
334
  row_id: options.rowId,
283
335
  checksum: options.checksum,
@@ -297,24 +349,27 @@ export abstract class PersistedBatch {
297
349
  async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
298
350
  const startAt = performance.now();
299
351
  let flushedSomething = false;
352
+ const writes = this.db.createWriteBatch(session, { ordered: false });
300
353
  if (this.bucketDataCount > 0) {
301
354
  flushedSomething = true;
302
- await this.flushBucketData(session);
355
+ await this.queueBucketData(writes);
303
356
  }
304
357
  if (this.bucketParameters.length > 0) {
305
358
  flushedSomething = true;
306
- await this.flushBucketParameters(session);
359
+ this.queueBucketParameters(writes);
307
360
  }
308
361
  if (this.currentDataCount > 0) {
309
362
  flushedSomething = true;
310
- await this.flushCurrentData(session);
363
+ this.queueCurrentData(writes);
311
364
  }
312
365
 
313
366
  if (this.bucketStates.size > 0) {
314
367
  flushedSomething = true;
315
- await this.flushBucketStates(session);
368
+ this.queueBucketStates(writes);
316
369
  }
317
370
 
371
+ await writes.execute();
372
+
318
373
  if (flushedSomething) {
319
374
  const duration = Math.round(performance.now() - startAt);
320
375
  if (options?.oldestUncommittedChange != null) {
@@ -377,4 +432,5 @@ export interface BucketStateUpdate {
377
432
  lastOp: InternalOpId;
378
433
  incrementCount: number;
379
434
  incrementBytes: number;
435
+ incrementChunks: number;
380
436
  }
@@ -58,6 +58,10 @@ export abstract class BaseVersionedPowerSyncMongo {
58
58
  return this.upstream.notifyCheckpoint();
59
59
  }
60
60
 
61
+ createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }) {
62
+ return this.upstream.createWriteBatch(session, options);
63
+ }
64
+
61
65
  protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
62
66
  return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
63
67
  }
@@ -17,6 +17,7 @@ import {
17
17
  SyncRuleDocumentBase,
18
18
  WriteCheckpointDocument
19
19
  } from './models.js';
20
+ import { MongoWriteBatch } from './MongoWriteBatch.js';
20
21
  import {
21
22
  BucketDataDocumentV1,
22
23
  BucketParameterDocument,
@@ -24,8 +25,9 @@ import {
24
25
  CurrentDataDocument
25
26
  } from './v1/models.js';
26
27
  import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
27
- import { BucketDataDocumentV3 } from './v3/models.js';
28
- import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
28
+ import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
29
+ import type { CustomCheckpointRequestCollectionSpecifier } from './v3/VersionedPowerSyncMongoV3.js';
30
+ import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
29
31
 
30
32
  export interface PowerSyncMongoOptions {
31
33
  /**
@@ -53,6 +55,19 @@ export class PowerSyncMongo {
53
55
 
54
56
  readonly client: mongo.MongoClient;
55
57
  readonly db: mongo.Db;
58
+ private get clientBulkWriteSupport(): boolean {
59
+ // The driver exposes topology at runtime, but omits this internal property
60
+ // from its public types. Reuse its handshake results without another command.
61
+ const client = this.client as mongo.MongoClient & { topology?: { description: mongo.TopologyDescription } };
62
+ const servers = [...(client.topology?.description.servers.values() ?? [])];
63
+ // MongoDB 8.0 introduced client bulkWrite at wire version 25. Unknown / mixed-version
64
+ // topologies use collection writes until all known servers support it.
65
+ return servers.length > 0 && servers.every((server) => server.maxWireVersion >= 25);
66
+ }
67
+
68
+ createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }): MongoWriteBatch {
69
+ return new MongoWriteBatch(this.client, this.clientBulkWriteSupport, session, options);
70
+ }
56
71
 
57
72
  constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions) {
58
73
  this.client = client;
@@ -137,6 +152,20 @@ export class PowerSyncMongo {
137
152
  return `source_table_${replicationStreamId}`;
138
153
  }
139
154
 
155
+ customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
156
+ return `custom_checkpoint_requests_${options.replicationStreamId}_${options.eventId}`;
157
+ }
158
+
159
+ async listCustomCheckpointRequestCollections(
160
+ replicationStreamId?: number
161
+ ): Promise<mongo.Collection<CustomCheckpointRequestDocumentV3>[]> {
162
+ const prefix =
163
+ replicationStreamId == null
164
+ ? 'custom_checkpoint_requests_'
165
+ : `custom_checkpoint_requests_${replicationStreamId}_`;
166
+ return this.collectionsByPrefix(prefix);
167
+ }
168
+
140
169
  async listSourceTableCollections(
141
170
  replicationStreamId?: number
142
171
  ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
@@ -178,6 +207,10 @@ export class PowerSyncMongo {
178
207
  await this.locks.deleteMany({});
179
208
  await this.bucket_state.deleteMany({});
180
209
  await this.custom_write_checkpoints.deleteMany({});
210
+ await this.db.collection(OBJECT_STORAGE_USAGE_COLLECTION).deleteMany({});
211
+ for (const collection of await this.listCustomCheckpointRequestCollections()) {
212
+ await collection.drop();
213
+ }
181
214
  }
182
215
 
183
216
  /**
@@ -74,6 +74,12 @@ export interface BucketDataProperties {
74
74
  op: OpType;
75
75
  source_table?: bson.ObjectId;
76
76
  source_key?: ReplicaId;
77
+ /**
78
+ * Stable identifier for the source record in the client protocol.
79
+ *
80
+ * This is derived from (source_table, source_key), so persistence is optional but preferred.
81
+ */
82
+ subkey?: string;
77
83
  table?: string;
78
84
  row_id?: string;
79
85
  checksum: bigint;
@@ -215,6 +221,34 @@ export interface SyncRuleDocumentBase {
215
221
  } | null;
216
222
 
217
223
  storage_version?: number;
224
+
225
+ /** Optional operator-supplied label identifying the sync config version. */
226
+ version_label?: string | null;
227
+
228
+ /**
229
+ * Incremental parameter compaction state for the replication stream.
230
+ *
231
+ * Operation ids are allocated across all parameter indexes of a stream, so this is
232
+ * stream-level state, shared by all sync configs of the stream.
233
+ */
234
+ parameter_compaction?: {
235
+ /**
236
+ * The exclusive operation-id boundary through which every parameter index in this stream has
237
+ * been compacted.
238
+ */
239
+ compacted_before: InternalOpId;
240
+
241
+ /**
242
+ * Parameter entries below this boundary may no longer be available for checkpoint change
243
+ * detection, since compaction may have deleted them.
244
+ *
245
+ * This is advanced before the first delete of a compaction pass, while
246
+ * {@link compacted_before} is only advanced after every delete of the pass completed.
247
+ * A checkpoint transition starting below this boundary must conservatively invalidate all
248
+ * parameter buckets, since the individual changes may no longer be available.
249
+ */
250
+ checkpoint_changes_invalid_before?: InternalOpId;
251
+ };
218
252
  }
219
253
 
220
254
  export interface SyncRuleCheckpointFields {
@@ -1,3 +1,4 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
2
3
  import { ColumnDescriptor, InternalOpId, SourceTable, storage } from '@powersync/service-core';
3
4
  import { HydratedSyncConfig } from '@powersync/service-sync-rules';
@@ -50,6 +51,43 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
50
51
  // No-op for V1: source records live in a shared collection.
51
52
  }
52
53
 
54
+ protected override async batchCreateCustomWriteCheckpoints(
55
+ session: mongo.ClientSession,
56
+ opId: InternalOpId
57
+ ): Promise<void> {
58
+ // Only the final checkpoint per user/stream is visible after this transaction.
59
+ const checkpoints = new Map(
60
+ this.write_checkpoint_batch.map((checkpoint) => [
61
+ JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
62
+ checkpoint
63
+ ])
64
+ );
65
+ await this.db.custom_write_checkpoints.bulkWrite(
66
+ [...checkpoints.values()].map((checkpointOptions) => {
67
+ const set: Record<string, unknown> = {
68
+ checkpoint: checkpointOptions.checkpoint,
69
+ sync_rules_id: checkpointOptions.sync_rules_id,
70
+ op_id: opId
71
+ };
72
+ if (checkpointOptions.checkpoint_requested_at != null) {
73
+ set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
74
+ }
75
+
76
+ return {
77
+ updateOne: {
78
+ filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
79
+ update: {
80
+ $set: set,
81
+ ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
82
+ },
83
+ upsert: true
84
+ }
85
+ };
86
+ }),
87
+ { session, ordered: false }
88
+ );
89
+ }
90
+
53
91
  async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
54
92
  const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
55
93
  const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
@@ -431,7 +469,9 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
431
469
  const ids = tables.map((table) => mongoTableId(table.id));
432
470
 
433
471
  await this.withTransaction(async () => {
434
- await this.db.sourceTablesV1(this.replicationStreamId).updateMany(
472
+ const writes = this.db.createWriteBatch(session, { ordered: false });
473
+ writes.updateMany(
474
+ this.db.sourceTablesV1(this.replicationStreamId),
435
475
  { _id: { $in: ids } },
436
476
  {
437
477
  $set: {
@@ -440,12 +480,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
440
480
  $unset: {
441
481
  snapshot_status: 1
442
482
  }
443
- },
444
- { session }
483
+ }
445
484
  );
446
485
 
447
486
  if (no_checkpoint_before_lsn != null) {
448
- await this.db.sync_rules.updateOne(
487
+ writes.updateOne(
488
+ this.db.sync_rules,
449
489
  {
450
490
  _id: this.replicationStreamId
451
491
  },
@@ -456,10 +496,10 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
456
496
  $max: {
457
497
  no_checkpoint_before: no_checkpoint_before_lsn
458
498
  }
459
- },
460
- { session: this.session }
499
+ }
461
500
  );
462
501
  }
502
+ await writes.execute();
463
503
  });
464
504
  return tables.map((table) => {
465
505
  const copy = table.clone();