@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -13,8 +13,10 @@ import {
|
|
|
13
13
|
UpsertCurrentDataOptions
|
|
14
14
|
} from '../common/PersistedBatch.js';
|
|
15
15
|
import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
|
|
16
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
16
17
|
import { serializeBucketData } from './bucket-format.js';
|
|
17
18
|
import { chunkBucketData } from './chunking.js';
|
|
19
|
+
import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js';
|
|
18
20
|
import {
|
|
19
21
|
BucketDataDocumentV3,
|
|
20
22
|
BucketStateDocumentV3,
|
|
@@ -24,12 +26,21 @@ import {
|
|
|
24
26
|
taggedBucketParameterDocumentToTagged
|
|
25
27
|
} from './models.js';
|
|
26
28
|
import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
|
|
29
|
+
import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
|
|
27
30
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
28
31
|
|
|
32
|
+
interface SourceRecordWrite {
|
|
33
|
+
sourceTableId: bson.ObjectId;
|
|
34
|
+
operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3>;
|
|
35
|
+
}
|
|
36
|
+
|
|
29
37
|
export class PersistedBatchV3 extends PersistedBatch {
|
|
30
|
-
|
|
38
|
+
// Upserts and soft deletes supply the complete source-record state, including
|
|
39
|
+
// pending_delete. Keep the final state per key, without removing any history.
|
|
40
|
+
currentData = new Map<string, SourceRecordWrite>();
|
|
31
41
|
sourceTablePendingDeletes = new Map<string, InternalOpId>();
|
|
32
42
|
protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
|
|
43
|
+
protected readonly objectStorageUsage?: ObjectStorageUsage;
|
|
33
44
|
|
|
34
45
|
declare protected readonly db: VersionedPowerSyncMongoV3;
|
|
35
46
|
|
|
@@ -43,6 +54,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
43
54
|
super(db, group_id, mapping, writtenSize, options);
|
|
44
55
|
if (this.objectStorage) {
|
|
45
56
|
this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
|
|
57
|
+
this.objectStorageUsage = new ObjectStorageUsage(this.db, this.group_id, this.objectStorageUsageWriterId);
|
|
46
58
|
}
|
|
47
59
|
}
|
|
48
60
|
|
|
@@ -127,7 +139,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
127
139
|
}
|
|
128
140
|
|
|
129
141
|
hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) {
|
|
130
|
-
this.currentData.
|
|
142
|
+
this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
|
|
131
143
|
sourceTableId,
|
|
132
144
|
operation: {
|
|
133
145
|
deleteOne: {
|
|
@@ -143,7 +155,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
143
155
|
replicaId: storage.ReplicaId,
|
|
144
156
|
checkpointGreaterThan: InternalOpId
|
|
145
157
|
) {
|
|
146
|
-
this.currentData.
|
|
158
|
+
this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
|
|
147
159
|
sourceTableId,
|
|
148
160
|
operation: {
|
|
149
161
|
updateOne: {
|
|
@@ -191,7 +203,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
191
203
|
};
|
|
192
204
|
});
|
|
193
205
|
|
|
194
|
-
this.currentData.
|
|
206
|
+
this.currentData.set(this.currentDataKey(values.sourceTableId, values.replicaId), {
|
|
195
207
|
sourceTableId: values.sourceTableId,
|
|
196
208
|
operation: {
|
|
197
209
|
updateOne: {
|
|
@@ -212,12 +224,12 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
212
224
|
}
|
|
213
225
|
|
|
214
226
|
protected get currentDataCount() {
|
|
215
|
-
return this.currentData.
|
|
227
|
+
return this.currentData.size;
|
|
216
228
|
}
|
|
217
229
|
|
|
218
230
|
// Flush methods
|
|
219
231
|
|
|
220
|
-
protected async
|
|
232
|
+
protected async queueBucketData(writes: MongoWriteBatch) {
|
|
221
233
|
const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
|
|
222
234
|
for (const document of this.bucketData) {
|
|
223
235
|
const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
|
|
@@ -225,17 +237,17 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
225
237
|
operationsByDefinition.set(document.bucketKey.definitionId, existing);
|
|
226
238
|
}
|
|
227
239
|
|
|
228
|
-
let uploadCount = 0;
|
|
229
240
|
const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
|
|
230
241
|
const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
|
|
231
|
-
const lifecycle = this.
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
const createInserts: (() => Promise<mongo.AnyBulkWriteOperation<BucketDataDocumentV3>>)[] = [];
|
|
242
|
+
const lifecycle = this.objectStorageLifecycle;
|
|
243
|
+
type BucketDataInsert = { insertOne: { document: BucketDataDocumentV3 } };
|
|
244
|
+
const createInserts: (() => Promise<BucketDataInsert>)[] = [];
|
|
235
245
|
|
|
236
246
|
for (const [bucket, ops] of operationsByBucket) {
|
|
247
|
+
this.resetBucketPersistedBytes(definitionId, bucket);
|
|
237
248
|
for (const chunk of chunkBucketData(ops)) {
|
|
238
249
|
const serialized = serializeBucketData(bucket, chunk);
|
|
250
|
+
this.incrementBucketPersistedChunk(definitionId, bucket, serialized.size);
|
|
239
251
|
if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
|
|
240
252
|
createInserts.push(async () => ({
|
|
241
253
|
insertOne: {
|
|
@@ -245,13 +257,12 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
245
257
|
continue;
|
|
246
258
|
}
|
|
247
259
|
|
|
248
|
-
uploadCount += 1;
|
|
249
260
|
createInserts.push(async () => {
|
|
250
261
|
const minOp = chunk[0].o;
|
|
251
262
|
const maxOp = chunk[chunk.length - 1].o;
|
|
252
263
|
const { ops: bucketOps, ...metadata } = serialized;
|
|
253
264
|
const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
|
|
254
|
-
const { fileSize } = await lifecycle.bucketData.store(path, bucketOps
|
|
265
|
+
const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!, { signal: this.signal });
|
|
255
266
|
return {
|
|
256
267
|
insertOne: {
|
|
257
268
|
document: {
|
|
@@ -281,19 +292,29 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
281
292
|
// S3ObjectStorage applies one shared concurrency limit across all callers,
|
|
282
293
|
// so replication can schedule its uploads together without creating a
|
|
283
294
|
// separate limiter here.
|
|
284
|
-
const
|
|
285
|
-
|
|
286
|
-
|
|
295
|
+
const bucketWrites = await createAllInserts();
|
|
296
|
+
|
|
297
|
+
const usageDeltas = this.objectStorageUsage == null ? undefined : new Map<BucketDefinitionId, bigint>();
|
|
298
|
+
for (const { definitionId, inserts } of bucketWrites) {
|
|
299
|
+
if (usageDeltas != null) {
|
|
300
|
+
const delta = inserts.reduce(
|
|
301
|
+
(sum, operation) => sum + ObjectStorageUsage.bytes(operation.insertOne.document),
|
|
302
|
+
0n
|
|
303
|
+
);
|
|
304
|
+
if (delta !== 0n) {
|
|
305
|
+
usageDeltas.set(definitionId, delta);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
287
308
|
if (inserts.length > 0) {
|
|
288
|
-
|
|
289
|
-
session,
|
|
290
|
-
ordered: false
|
|
291
|
-
});
|
|
309
|
+
writes.bulkWriteUnordered(this.db.bucketData(this.group_id, definitionId), inserts);
|
|
292
310
|
}
|
|
293
311
|
}
|
|
312
|
+
if (usageDeltas != null) {
|
|
313
|
+
this.objectStorageUsage!.applyDeltas(usageDeltas, writes);
|
|
314
|
+
}
|
|
294
315
|
}
|
|
295
316
|
|
|
296
|
-
protected
|
|
317
|
+
protected queueBucketParameters(writes: MongoWriteBatch): void {
|
|
297
318
|
const operationsByIndex = new Map<string, typeof this.bucketParameters>();
|
|
298
319
|
for (const document of this.bucketParameters) {
|
|
299
320
|
const existing = operationsByIndex.get(document.index) ?? [];
|
|
@@ -302,23 +323,20 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
302
323
|
}
|
|
303
324
|
|
|
304
325
|
for (const [indexId, documents] of operationsByIndex.entries()) {
|
|
305
|
-
|
|
326
|
+
writes.bulkWriteUnordered(
|
|
327
|
+
this.db.parameterIndex(this.group_id, indexId),
|
|
306
328
|
documents.map((document) => ({
|
|
307
329
|
insertOne: {
|
|
308
330
|
document: taggedBucketParameterDocumentToTagged(document)
|
|
309
331
|
}
|
|
310
|
-
}))
|
|
311
|
-
{
|
|
312
|
-
session,
|
|
313
|
-
ordered: false
|
|
314
|
-
}
|
|
332
|
+
}))
|
|
315
333
|
);
|
|
316
334
|
}
|
|
317
335
|
}
|
|
318
336
|
|
|
319
|
-
protected
|
|
320
|
-
const operationsBySourceTable = new Map<string,
|
|
321
|
-
for (const operation of this.currentData) {
|
|
337
|
+
protected queueCurrentData(writes: MongoWriteBatch): void {
|
|
338
|
+
const operationsBySourceTable = new Map<string, SourceRecordWrite[]>();
|
|
339
|
+
for (const operation of this.currentData.values()) {
|
|
322
340
|
const sourceTableId = operation.sourceTableId.toHexString();
|
|
323
341
|
const existing = operationsBySourceTable.get(sourceTableId) ?? [];
|
|
324
342
|
existing.push(operation);
|
|
@@ -341,33 +359,31 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
341
359
|
});
|
|
342
360
|
|
|
343
361
|
if (sourceTableUpdates.length > 0) {
|
|
344
|
-
|
|
362
|
+
writes.bulkWriteUnordered(this.db.sourceTables(this.group_id), sourceTableUpdates);
|
|
345
363
|
}
|
|
346
364
|
|
|
347
365
|
for (const operations of operationsBySourceTable.values()) {
|
|
348
366
|
const sourceTableId = operations[0]!.sourceTableId;
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
session,
|
|
353
|
-
ordered: true
|
|
354
|
-
}
|
|
367
|
+
writes.bulkWriteUnordered(
|
|
368
|
+
this.db.sourceRecords(this.group_id, sourceTableId),
|
|
369
|
+
operations.map((entry) => entry.operation)
|
|
355
370
|
);
|
|
356
371
|
}
|
|
357
372
|
}
|
|
358
373
|
|
|
359
|
-
protected
|
|
360
|
-
|
|
361
|
-
session,
|
|
362
|
-
ordered: false
|
|
363
|
-
});
|
|
374
|
+
protected queueBucketStates(writes: MongoWriteBatch): void {
|
|
375
|
+
writes.bulkWriteUnordered(this.db.bucketState(this.group_id), this.getBucketStateUpdates());
|
|
364
376
|
}
|
|
365
377
|
|
|
366
378
|
protected resetCurrentData() {
|
|
367
|
-
this.currentData
|
|
379
|
+
this.currentData.clear();
|
|
368
380
|
this.sourceTablePendingDeletes.clear();
|
|
369
381
|
}
|
|
370
382
|
|
|
383
|
+
private currentDataKey(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): string {
|
|
384
|
+
return Buffer.from(bson.serialize({ t: sourceTableId, k: replicaId })).toString('base64');
|
|
385
|
+
}
|
|
386
|
+
|
|
371
387
|
private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocumentV3>[] {
|
|
372
388
|
return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => {
|
|
373
389
|
if (state.definitionId == null) {
|
|
@@ -381,15 +397,42 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
381
397
|
b: state.bucket
|
|
382
398
|
}
|
|
383
399
|
},
|
|
384
|
-
update
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
400
|
+
// A pipeline update makes initialisation and scheduling one atomic
|
|
401
|
+
// writer operation. In particular, a later write cannot move an
|
|
402
|
+
// already-due compact check into the future.
|
|
403
|
+
update: [
|
|
404
|
+
{
|
|
405
|
+
$set: {
|
|
406
|
+
last_op: state.lastOp,
|
|
407
|
+
bucket_stats: {
|
|
408
|
+
count: { $add: [{ $ifNull: ['$bucket_stats.count', 0] }, state.incrementCount] },
|
|
409
|
+
bytes: { $add: [{ $ifNull: ['$bucket_stats.bytes', 0n] }, BigInt(state.incrementBytes)] },
|
|
410
|
+
chunks: { $add: [{ $ifNull: ['$bucket_stats.chunks', 0] }, state.incrementChunks] }
|
|
411
|
+
},
|
|
412
|
+
first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] },
|
|
413
|
+
next_compact_check: {
|
|
414
|
+
$let: {
|
|
415
|
+
vars: {
|
|
416
|
+
requested: {
|
|
417
|
+
$dateAdd: {
|
|
418
|
+
startDate: '$$NOW',
|
|
419
|
+
unit: 'millisecond',
|
|
420
|
+
amount: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
},
|
|
424
|
+
in: {
|
|
425
|
+
$cond: [
|
|
426
|
+
{ $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] },
|
|
427
|
+
'$next_compact_check',
|
|
428
|
+
'$$requested'
|
|
429
|
+
]
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
391
434
|
}
|
|
392
|
-
|
|
435
|
+
],
|
|
393
436
|
upsert: true
|
|
394
437
|
}
|
|
395
438
|
} satisfies mongo.AnyBulkWriteOperation<BucketStateDocumentV3>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
|
|
2
|
+
import { BucketDefinitionId, EventDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
|
|
3
3
|
import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
|
|
4
4
|
import { CommonSourceTableDocument } from '../models.js';
|
|
5
5
|
import {
|
|
@@ -7,11 +7,19 @@ import {
|
|
|
7
7
|
BucketParameterDocumentV3,
|
|
8
8
|
BucketStateDocumentV3,
|
|
9
9
|
CurrentDataDocumentV3,
|
|
10
|
+
CustomCheckpointRequestDocumentV3,
|
|
10
11
|
ObjectStorageDeletionMarker,
|
|
12
|
+
ObjectStorageUsageDocument,
|
|
11
13
|
SourceTableDocumentV3,
|
|
12
14
|
SyncConfigDefinition
|
|
13
15
|
} from './models.js';
|
|
14
16
|
|
|
17
|
+
export const OBJECT_STORAGE_USAGE_COLLECTION = 'object_storage_usage';
|
|
18
|
+
export type CustomCheckpointRequestCollectionSpecifier = {
|
|
19
|
+
replicationStreamId: number;
|
|
20
|
+
eventId: EventDefinitionId;
|
|
21
|
+
};
|
|
22
|
+
|
|
15
23
|
export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
16
24
|
constructor(
|
|
17
25
|
upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
|
|
@@ -74,6 +82,42 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
|
74
82
|
return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
|
|
75
83
|
}
|
|
76
84
|
|
|
85
|
+
get objectStorageUsage(): mongo.Collection<ObjectStorageUsageDocument> {
|
|
86
|
+
return this.db.collection<ObjectStorageUsageDocument>(OBJECT_STORAGE_USAGE_COLLECTION);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
customCheckpointRequests(options: CustomCheckpointRequestCollectionSpecifier) {
|
|
90
|
+
const collectionName = this.customCheckpointRequestCollectionName(options);
|
|
91
|
+
return this.db.collection<CustomCheckpointRequestDocumentV3>(collectionName);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async initializeCustomCheckpointRequestsCollection(options: CustomCheckpointRequestCollectionSpecifier) {
|
|
95
|
+
// createIndexes also creates the collection on first use. Callers invoke
|
|
96
|
+
// this lazily before the transaction which writes the first checkpoint.
|
|
97
|
+
await this.customCheckpointRequests(options).createIndexes([
|
|
98
|
+
{
|
|
99
|
+
key: { user_id: 1 },
|
|
100
|
+
name: 'user_unique',
|
|
101
|
+
unique: true
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
// Find checkpoint changes over an operation-id range.
|
|
105
|
+
key: { op_id: 1 },
|
|
106
|
+
name: 'op_id'
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
// Limit retention scans to temporary client-requested checkpoints.
|
|
110
|
+
key: { checkpoint_requested_at: 1 },
|
|
111
|
+
name: 'checkpoint_requested_at',
|
|
112
|
+
partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
|
|
113
|
+
}
|
|
114
|
+
]);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
listCustomCheckpointRequestCollections(replicationStreamId: number) {
|
|
118
|
+
return this.upstream.listCustomCheckpointRequestCollections(replicationStreamId);
|
|
119
|
+
}
|
|
120
|
+
|
|
77
121
|
listBucketDataCollections(replicationStreamId: number) {
|
|
78
122
|
return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
|
|
79
123
|
}
|
|
@@ -117,11 +161,18 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
|
117
161
|
},
|
|
118
162
|
{ name: 'bucket_updates', unique: true }
|
|
119
163
|
);
|
|
164
|
+
// V3 workers claim only buckets that have reached their scheduled check
|
|
165
|
+
// time, keeping scheduled scans proportional to pending work.
|
|
120
166
|
await bucketState.createIndex(
|
|
167
|
+
{ next_compact_check: 1 },
|
|
121
168
|
{
|
|
122
|
-
'
|
|
123
|
-
|
|
124
|
-
|
|
169
|
+
name: 'next_compact_check',
|
|
170
|
+
partialFilterExpression: { next_compact_check: { $exists: true } }
|
|
171
|
+
}
|
|
125
172
|
);
|
|
126
173
|
}
|
|
174
|
+
|
|
175
|
+
customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
|
|
176
|
+
return this.upstream.customCheckpointRequestCollectionName(options);
|
|
177
|
+
}
|
|
127
178
|
}
|
|
@@ -32,6 +32,7 @@ export function serializeBucketData(
|
|
|
32
32
|
op: op.op,
|
|
33
33
|
source_table: op.source_table,
|
|
34
34
|
source_key: op.source_key,
|
|
35
|
+
subkey: op.subkey,
|
|
35
36
|
table: op.table,
|
|
36
37
|
row_id: op.row_id,
|
|
37
38
|
checksum: op.checksum,
|
|
@@ -78,6 +79,7 @@ export function* loadBucketDataDocument(
|
|
|
78
79
|
op: op.op,
|
|
79
80
|
source_table: op.source_table,
|
|
80
81
|
source_key: op.source_key,
|
|
82
|
+
subkey: op.subkey,
|
|
81
83
|
table: op.table,
|
|
82
84
|
row_id: op.row_id,
|
|
83
85
|
checksum: op.checksum,
|