@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -0,0 +1,172 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+
3
+ /**
4
+ * Explicitly groups writes with no intervening reads or result dependencies.
5
+ * Callers retain responsibility for transaction and ordering boundaries.
6
+ *
7
+ * On MongoDB 8.0+, this uses the client-level bulkWrite API.
8
+ *
9
+ * On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
10
+ * This is less efficient, but produces the same results.
11
+ */
12
+ export class MongoWriteBatch {
13
+ private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
14
+ private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
15
+ private readonly fallback: (() => Promise<unknown>)[] = [];
16
+
17
+ private readonly ordered: boolean;
18
+
19
+ constructor(
20
+ private readonly client: mongo.MongoClient,
21
+ private readonly supportsClientBulkWrite: boolean,
22
+ private readonly session: mongo.ClientSession | undefined,
23
+ options: { ordered: boolean }
24
+ ) {
25
+ this.ordered = options.ordered;
26
+ }
27
+
28
+ /**
29
+ * Perform a collection-level bulkWrite.
30
+ *
31
+ * On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
32
+ * The driver may split commands to respect server batch limits.
33
+ *
34
+ * On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
35
+ * Operations within this collection bulk must therefore be independent of their order.
36
+ */
37
+ bulkWriteUnordered<T extends mongo.Document>(
38
+ collection: mongo.Collection<T>,
39
+ operations: mongo.AnyBulkWriteOperation<T>[]
40
+ ): void {
41
+ if (operations.length === 0) return;
42
+ if (!this.supportsClientBulkWrite) {
43
+ this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
44
+ return;
45
+ }
46
+ for (const operation of operations) {
47
+ // Collection and client models contain the same operation fields, with a
48
+ // different envelope. Keep the schema checked at the collection boundary.
49
+ const [name, model] = Object.entries(operation)[0];
50
+ this.operations.push({ ...model, name, namespace: collection.namespace });
51
+ }
52
+ }
53
+
54
+ insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
55
+ if (!this.supportsClientBulkWrite) {
56
+ this.fallback.push(() => collection.insertOne(document, { session: this.session }));
57
+ return;
58
+ }
59
+ this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
60
+ }
61
+
62
+ insertMany<T extends mongo.Document>(
63
+ collection: mongo.Collection<T>,
64
+ documents: mongo.OptionalUnlessRequiredId<T>[]
65
+ ) {
66
+ if (documents.length === 0) return;
67
+ if (!this.supportsClientBulkWrite) {
68
+ this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
69
+ return;
70
+ }
71
+ for (const document of documents) {
72
+ this.addModel<T>({
73
+ name: 'insertOne',
74
+ namespace: collection.namespace,
75
+ document: document as mongo.OptionalId<T>
76
+ });
77
+ }
78
+ }
79
+
80
+ deleteMany<T extends mongo.Document>(
81
+ collection: mongo.Collection<T>,
82
+ filter: mongo.Filter<T>,
83
+ checkDeletedCount?: (count: number) => void
84
+ ) {
85
+ if (checkDeletedCount) {
86
+ // Client-bulk results are checked after all writes execute. A failed
87
+ // invariant must roll back the entire batch, not leave later writes visible.
88
+ if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
89
+ }
90
+ if (!this.supportsClientBulkWrite) {
91
+ this.fallback.push(async () => {
92
+ const result = await collection.deleteMany(filter, { session: this.session });
93
+ checkDeletedCount?.(result.deletedCount);
94
+ });
95
+ return;
96
+ }
97
+ if (checkDeletedCount) {
98
+ this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
99
+ }
100
+ this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
101
+ }
102
+
103
+ updateOne<T extends mongo.Document>(
104
+ collection: mongo.Collection<T>,
105
+ filter: mongo.Filter<T>,
106
+ update: mongo.UpdateFilter<T> | mongo.Document[],
107
+ options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
108
+ ) {
109
+ if (!this.supportsClientBulkWrite) {
110
+ this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
111
+ return;
112
+ }
113
+ this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
114
+ }
115
+
116
+ updateMany<T extends mongo.Document>(
117
+ collection: mongo.Collection<T>,
118
+ filter: mongo.Filter<T>,
119
+ update: mongo.UpdateFilter<T> | mongo.Document[],
120
+ options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
121
+ ) {
122
+ if (!this.supportsClientBulkWrite) {
123
+ this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
124
+ return;
125
+ }
126
+ this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
127
+ }
128
+
129
+ private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
130
+ // The collection-specific methods validate the schema before erasing it here.
131
+ this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
132
+ }
133
+
134
+ /**
135
+ * Execute the queued writes.
136
+ *
137
+ * On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
138
+ * The command may be split if there are a large number of operations.
139
+ *
140
+ * On earlier MongoDB versions, this executes each queued operation individually.
141
+ *
142
+ * If no operations were queued, this is a no-op.
143
+ */
144
+ async execute(): Promise<void> {
145
+ if (!this.supportsClientBulkWrite) {
146
+ // Sessions cannot execute concurrent operations within a transaction.
147
+ for (const write of this.fallback) await write();
148
+ return;
149
+ }
150
+ if (this.operations.length === 0) return;
151
+
152
+ try {
153
+ const result = await this.client.bulkWrite(this.operations, {
154
+ session: this.session,
155
+ ordered: this.ordered,
156
+ verboseResults: this.deleteChecks.length > 0
157
+ });
158
+ for (const { index, check } of this.deleteChecks) {
159
+ const deleted = result.deleteResults?.get(index);
160
+ if (deleted == null) throw new Error('Missing client bulk delete result');
161
+ check(deleted.deletedCount);
162
+ }
163
+ } catch (error) {
164
+ // The driver wraps command failures without copying their error labels.
165
+ // Preserve withTransaction's retry handling for e.g. write conflicts.
166
+ if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
167
+ throw error.cause;
168
+ }
169
+ throw error;
170
+ }
171
+ }
172
+ }
@@ -1,29 +1,31 @@
1
- import { mongo } from '@powersync/lib-service-mongodb';
2
1
  import * as framework from '@powersync/lib-services-framework';
3
- import { GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
2
+ import { GetCheckpointChangesOptions, storage } from '@powersync/service-core';
4
3
  import { VersionedPowerSyncMongo } from './db.js';
5
4
 
6
5
  export type MongoCheckpointAPIOptions = {
7
6
  db: VersionedPowerSyncMongo;
8
- mode: storage.WriteCheckpointMode;
9
- sync_rules_id: number;
7
+ writeCheckpointMode: storage.WriteCheckpointModeConfig;
8
+ replicationStreamId: number;
10
9
  };
11
10
 
12
11
  export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
13
12
  readonly db: VersionedPowerSyncMongo;
13
+ readonly replicationStreamId: number;
14
+
14
15
  private _mode: storage.WriteCheckpointMode;
15
16
 
16
17
  constructor(options: MongoCheckpointAPIOptions) {
17
18
  this.db = options.db;
18
- this._mode = options.mode;
19
+ this._mode = options.writeCheckpointMode.mode;
20
+ this.replicationStreamId = options.replicationStreamId;
19
21
  }
20
22
 
21
23
  get writeCheckpointMode() {
22
24
  return this._mode;
23
25
  }
24
26
 
25
- setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
26
- this._mode = mode;
27
+ setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
28
+ this._mode = config.mode;
27
29
  }
28
30
 
29
31
  async createManagedWriteCheckpoints(
@@ -93,7 +95,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
93
95
  },
94
96
  upsert: true
95
97
  }
96
- }))
98
+ })),
99
+ { ordered: false }
97
100
  );
98
101
 
99
102
  const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
@@ -178,7 +181,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
178
181
  upsert: true
179
182
  }
180
183
  };
181
- })
184
+ }),
185
+ { ordered: false }
182
186
  );
183
187
 
184
188
  // Fetch the final ids separately so stale requests can still return the
@@ -247,6 +251,7 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
247
251
  }
248
252
 
249
253
  protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) {
254
+ // Legacy V1/V2 implementation.
250
255
  const { user_id, sync_rules_id } = filters;
251
256
  const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
252
257
  user_id,
@@ -301,8 +306,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
301
306
  };
302
307
  }
303
308
 
304
- private async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
309
+ protected async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
305
310
  const limit = 1000;
311
+ // Legacy V1/V2 implementation.
306
312
  const changes = await this.db.custom_write_checkpoints
307
313
  .find(
308
314
  {
@@ -332,39 +338,3 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
332
338
  };
333
339
  }
334
340
  }
335
-
336
- export async function batchCreateCustomWriteCheckpoints(
337
- db: VersionedPowerSyncMongo,
338
- session: mongo.ClientSession,
339
- checkpoints: storage.CustomWriteCheckpointOptions[],
340
- opId: InternalOpId
341
- ): Promise<void> {
342
- if (checkpoints.length == 0) {
343
- return;
344
- }
345
-
346
- await db.custom_write_checkpoints.bulkWrite(
347
- checkpoints.map((checkpointOptions) => {
348
- const set: Record<string, unknown> = {
349
- checkpoint: checkpointOptions.checkpoint,
350
- sync_rules_id: checkpointOptions.sync_rules_id,
351
- op_id: opId
352
- };
353
- if (checkpointOptions.checkpoint_requested_at != null) {
354
- set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
355
- }
356
-
357
- return {
358
- updateOne: {
359
- filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
360
- update: {
361
- $set: set,
362
- ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
363
- },
364
- upsert: true
365
- }
366
- };
367
- }),
368
- { session }
369
- );
370
- }
@@ -2,21 +2,28 @@ import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { storage } from '@powersync/service-core';
3
3
 
4
4
  /**
5
- * Update pipeline to update replication stream status, covering all storage versions.
5
+ * Update pipeline to stop a deploying replication stream, covering all storage versions.
6
6
  *
7
7
  * Roughly equivalent to:
8
8
  * $set: {
9
- * state: state,
10
- * 'sync_configs.$[].state': state
9
+ * state: STOP,
10
+ * 'sync_configs.$[].state': STOP
11
11
  * }
12
12
  *
13
13
  * The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
14
+ *
15
+ * This can be used for specific use cases:
16
+ * 1. Stop an existing PROCESSING stream, when it is being replaced by a new PROCESSING one.
17
+ * 2. Stop an existing ACTIVE or ERRORED stream, when it is being replaced by a new ACTIVE one.
18
+ *
19
+ * Do not use this when an ACTIVE stream transitions to ERRORED, and is being replaced by a new PROCESSING one.
20
+ * In that case, we need to maintain the ERRORED state, which this pipeline does not cover.
14
21
  */
15
- export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo.Document[] {
22
+ export function stopReplicationStreamPipeline(): mongo.Document[] {
16
23
  return [
17
24
  {
18
25
  $set: {
19
- state,
26
+ state: storage.SyncRuleState.STOP,
20
27
  sync_configs: {
21
28
  $cond: [
22
29
  { $isArray: '$sync_configs' },
@@ -25,7 +32,7 @@ export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo
25
32
  input: '$sync_configs',
26
33
  as: 'config',
27
34
  in: {
28
- $mergeObjects: ['$$config', { state }]
35
+ $mergeObjects: ['$$config', { state: storage.SyncRuleState.STOP }]
29
36
  }
30
37
  }
31
38
  },
@@ -1,9 +1,31 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { InternalOpId } from '@powersync/service-core';
2
+ import { InternalOpId, ReplicationCheckpoint } from '@powersync/service-core';
3
3
  import * as bson from 'bson';
4
4
 
5
- export interface MongoSyncBucketStorageCheckpoint {
5
+ export interface MongoSyncBucketStorageCheckpoint extends ReplicationCheckpoint {
6
6
  checkpoint: InternalOpId;
7
7
  snapshotTime: bson.Timestamp;
8
8
  clusterTime: mongo.ClusterTime;
9
+
10
+ /**
11
+ * The stream's `parameter_compaction.checkpoint_changes_invalid_before` boundary, read in the
12
+ * same snapshot as {@link checkpoint} and {@link snapshotTime}.
13
+ *
14
+ * Parameter compaction may have deleted parameter entries below this boundary, so checkpoint
15
+ * change detection cannot enumerate individual lookup changes from a checkpoint below it.
16
+ *
17
+ * 0n for streams that have never been compacted.
18
+ */
19
+ parameterChangesInvalidBefore: InternalOpId;
20
+ }
21
+
22
+ /**
23
+ * MongoDB-specific version of GetCheckpointChangesOptions.
24
+ *
25
+ * The next checkpoint carries the snapshot and invalidation boundary that change detection must
26
+ * be evaluated against. Only `checkpoint` is used from the previous one.
27
+ */
28
+ export interface MongoGetCheckpointChangesOptions {
29
+ lastCheckpoint: ReplicationCheckpoint;
30
+ nextCheckpoint: MongoSyncBucketStorageCheckpoint;
9
31
  }
@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
8
8
  import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
9
9
  import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
10
10
  import { MongoIdSequence } from '../MongoIdSequence.js';
11
+ import { MongoWriteBatch } from '../MongoWriteBatch.js';
11
12
  import type { VersionedPowerSyncMongo } from '../db.js';
12
13
  import { TaggedBucketParameterDocument } from '../models.js';
13
14
  import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
@@ -65,6 +66,11 @@ export interface PersistedBatchOptions {
65
66
  logger?: Logger;
66
67
  objectStorage?: ObjectStorage;
67
68
  inlineThresholdBytes?: number;
69
+ /**
70
+ * Aborts in-flight object storage uploads when replication stops.
71
+ */
72
+ signal?: AbortSignal;
73
+ objectStorageUsageWriterId?: string;
68
74
  }
69
75
 
70
76
  /**
@@ -81,6 +87,8 @@ export abstract class PersistedBatch {
81
87
 
82
88
  protected readonly objectStorage?: ObjectStorage;
83
89
  protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
90
+ protected readonly signal?: AbortSignal;
91
+ protected readonly objectStorageUsageWriterId?: string;
84
92
 
85
93
  /**
86
94
  * For debug logging only.
@@ -102,6 +110,8 @@ export abstract class PersistedBatch {
102
110
  this.currentSize = writtenSize;
103
111
  this.logger = options?.logger ?? defaultLogger;
104
112
  this.objectStorage = options?.objectStorage;
113
+ this.signal = options?.signal;
114
+ this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
105
115
  if (options?.inlineThresholdBytes != null) {
106
116
  this.inlineThresholdBytes = options.inlineThresholdBytes;
107
117
  }
@@ -124,7 +134,8 @@ export abstract class PersistedBatch {
124
134
  remaining_buckets.set(currentBucketKey(mapped), mapped);
125
135
  }
126
136
 
127
- const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)));
137
+ const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
138
+ const dchecksum = BigInt(utils.hashDelete(subkey));
128
139
 
129
140
  for (const evaluated of options.evaluated) {
130
141
  const definitionId = this.getBucketDefinitionId(evaluated.source);
@@ -159,6 +170,7 @@ export abstract class PersistedBatch {
159
170
  bucket: evaluated.bucket,
160
171
  sourceTableId: options.table.id,
161
172
  sourceKey: options.sourceKey,
173
+ subkey,
162
174
  table: evaluated.table,
163
175
  rowId: evaluated.id,
164
176
  checksum: BigInt(checksum),
@@ -181,6 +193,7 @@ export abstract class PersistedBatch {
181
193
  op_id,
182
194
  sourceTableId: options.table.id,
183
195
  sourceKey: options.sourceKey,
196
+ subkey,
184
197
  table: bucket.table,
185
198
  rowId: bucket.id,
186
199
  checksum: dchecksum
@@ -204,19 +217,27 @@ export abstract class PersistedBatch {
204
217
 
205
218
  protected abstract get currentDataCount(): number;
206
219
 
207
- protected abstract flushBucketData(session: mongo.ClientSession): Promise<void>;
220
+ protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
208
221
 
209
- protected abstract flushBucketParameters(session: mongo.ClientSession): Promise<void>;
222
+ protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
210
223
 
211
- protected abstract flushCurrentData(session: mongo.ClientSession): Promise<void>;
224
+ protected abstract queueCurrentData(writes: MongoWriteBatch): void;
212
225
 
213
- protected abstract flushBucketStates(session: mongo.ClientSession): Promise<void>;
226
+ protected abstract queueBucketStates(writes: MongoWriteBatch): void;
214
227
 
215
228
  protected abstract resetCurrentData(): void;
216
229
 
217
230
  protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
218
231
  protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
219
232
 
233
+ /**
234
+ * New bucket operations persist protocol subkeys during replication. Older
235
+ * V1 documents may not have one, so sync reads retain a fallback.
236
+ */
237
+ protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
238
+ return replicaIdToSubkey(table, key);
239
+ }
240
+
220
241
  protected get bucketDataCount(): number {
221
242
  return this.bucketData.length;
222
243
  }
@@ -234,17 +255,45 @@ export abstract class PersistedBatch {
234
255
  bucket,
235
256
  lastOp: op_id,
236
257
  incrementCount: 1,
237
- incrementBytes: bytes
258
+ incrementBytes: bytes,
259
+ incrementChunks: 0
238
260
  });
239
261
  }
240
262
  }
241
263
 
264
+ /**
265
+ * V3 persists operations in chunks. Keep this separate from incrementBucket:
266
+ * operation counts are known while evaluating rows, while chunk counts and
267
+ * exact persisted bytes are only known after the writer has chunked a flush.
268
+ */
269
+ protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
270
+ const key = `${definitionId ?? ''}:${bucket}`;
271
+ const existingState = this.bucketStates.get(key);
272
+ if (existingState != null) {
273
+ existingState.incrementChunks += 1;
274
+ existingState.incrementBytes += bytes;
275
+ }
276
+ }
277
+
278
+ /**
279
+ * V3's compactor calculates byte deltas from persisted chunk metadata.
280
+ * Replace the writer's per-operation estimate with those exact sizes before
281
+ * flushing the corresponding bucket state.
282
+ */
283
+ protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
284
+ const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
285
+ if (state != null) {
286
+ state.incrementBytes = 0;
287
+ }
288
+ }
289
+
242
290
  protected addBucketDataPut(options: {
243
291
  op_id: InternalOpId;
244
292
  bucketKey: BucketKey;
245
293
  bucket: string;
246
294
  sourceTableId: storage.SourceTable['id'];
247
295
  sourceKey: storage.ReplicaId;
296
+ subkey: string;
248
297
  table: string;
249
298
  rowId: string;
250
299
  checksum: bigint;
@@ -256,6 +305,7 @@ export abstract class PersistedBatch {
256
305
  op: 'PUT',
257
306
  source_table: mongoTableId(options.sourceTableId),
258
307
  source_key: options.sourceKey,
308
+ subkey: options.subkey,
259
309
  table: options.table,
260
310
  row_id: options.rowId,
261
311
  checksum: options.checksum,
@@ -268,6 +318,7 @@ export abstract class PersistedBatch {
268
318
  bucketKey: BucketKey;
269
319
  sourceTableId: storage.SourceTable['id'];
270
320
  sourceKey: storage.ReplicaId;
321
+ subkey: string;
271
322
  table: string;
272
323
  rowId: string;
273
324
  checksum: bigint;
@@ -278,6 +329,7 @@ export abstract class PersistedBatch {
278
329
  op: 'REMOVE',
279
330
  source_table: mongoTableId(options.sourceTableId),
280
331
  source_key: options.sourceKey,
332
+ subkey: options.subkey,
281
333
  table: options.table,
282
334
  row_id: options.rowId,
283
335
  checksum: options.checksum,
@@ -297,56 +349,59 @@ export abstract class PersistedBatch {
297
349
  async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
298
350
  const startAt = performance.now();
299
351
  let flushedSomething = false;
352
+ const writes = this.db.createWriteBatch(session, { ordered: false });
300
353
  if (this.bucketDataCount > 0) {
301
354
  flushedSomething = true;
302
- await this.flushBucketData(session);
355
+ await this.queueBucketData(writes);
303
356
  }
304
357
  if (this.bucketParameters.length > 0) {
305
358
  flushedSomething = true;
306
- await this.flushBucketParameters(session);
359
+ this.queueBucketParameters(writes);
307
360
  }
308
361
  if (this.currentDataCount > 0) {
309
362
  flushedSomething = true;
310
- await this.flushCurrentData(session);
363
+ this.queueCurrentData(writes);
311
364
  }
312
365
 
313
366
  if (this.bucketStates.size > 0) {
314
367
  flushedSomething = true;
315
- await this.flushBucketStates(session);
368
+ this.queueBucketStates(writes);
316
369
  }
317
370
 
371
+ await writes.execute();
372
+
318
373
  if (flushedSomething) {
319
374
  const duration = Math.round(performance.now() - startAt);
320
375
  if (options?.oldestUncommittedChange != null) {
321
376
  const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
322
377
 
323
378
  this.logger.info(
324
- `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${
379
+ `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
325
380
  this.currentDataCount
326
- } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
381
+ } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
327
382
  {
328
383
  flushed: {
329
384
  duration: duration,
330
385
  size: this.currentSize,
331
- bucket_data_count: this.bucketDataCount,
332
- parameter_data_count: this.bucketParameters.length,
333
- current_data_count: this.currentDataCount,
386
+ bucket_ops_count: this.bucketDataCount,
387
+ parameter_indexes_count: this.bucketParameters.length,
388
+ source_records_count: this.currentDataCount,
334
389
  replication_lag_seconds: replicationLag
335
390
  }
336
391
  }
337
392
  );
338
393
  } else {
339
394
  this.logger.info(
340
- `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${
395
+ `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
341
396
  this.currentDataCount
342
- } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
397
+ } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
343
398
  {
344
399
  flushed: {
345
400
  duration: duration,
346
401
  size: this.currentSize,
347
- bucket_data_count: this.bucketDataCount,
348
- parameter_data_count: this.bucketParameters.length,
349
- current_data_count: this.currentDataCount
402
+ bucket_ops_count: this.bucketDataCount,
403
+ parameter_indexes_count: this.bucketParameters.length,
404
+ source_records_count: this.currentDataCount
350
405
  }
351
406
  }
352
407
  );
@@ -377,4 +432,5 @@ export interface BucketStateUpdate {
377
432
  lastOp: InternalOpId;
378
433
  incrementCount: number;
379
434
  incrementBytes: number;
435
+ incrementChunks: number;
380
436
  }
@@ -58,6 +58,10 @@ export abstract class BaseVersionedPowerSyncMongo {
58
58
  return this.upstream.notifyCheckpoint();
59
59
  }
60
60
 
61
+ createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }) {
62
+ return this.upstream.createWriteBatch(session, options);
63
+ }
64
+
61
65
  protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
62
66
  return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
63
67
  }