@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Explicitly groups writes with no intervening reads or result dependencies.
|
|
5
|
+
* Callers retain responsibility for transaction and ordering boundaries.
|
|
6
|
+
*
|
|
7
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API.
|
|
8
|
+
*
|
|
9
|
+
* On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
|
|
10
|
+
* This is less efficient, but produces the same results.
|
|
11
|
+
*/
|
|
12
|
+
export class MongoWriteBatch {
|
|
13
|
+
private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
|
|
14
|
+
private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
|
|
15
|
+
private readonly fallback: (() => Promise<unknown>)[] = [];
|
|
16
|
+
|
|
17
|
+
private readonly ordered: boolean;
|
|
18
|
+
|
|
19
|
+
constructor(
|
|
20
|
+
private readonly client: mongo.MongoClient,
|
|
21
|
+
private readonly supportsClientBulkWrite: boolean,
|
|
22
|
+
private readonly session: mongo.ClientSession | undefined,
|
|
23
|
+
options: { ordered: boolean }
|
|
24
|
+
) {
|
|
25
|
+
this.ordered = options.ordered;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Perform a collection-level bulkWrite.
|
|
30
|
+
*
|
|
31
|
+
* On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
|
|
32
|
+
* The driver may split commands to respect server batch limits.
|
|
33
|
+
*
|
|
34
|
+
* On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
|
|
35
|
+
* Operations within this collection bulk must therefore be independent of their order.
|
|
36
|
+
*/
|
|
37
|
+
bulkWriteUnordered<T extends mongo.Document>(
|
|
38
|
+
collection: mongo.Collection<T>,
|
|
39
|
+
operations: mongo.AnyBulkWriteOperation<T>[]
|
|
40
|
+
): void {
|
|
41
|
+
if (operations.length === 0) return;
|
|
42
|
+
if (!this.supportsClientBulkWrite) {
|
|
43
|
+
this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
for (const operation of operations) {
|
|
47
|
+
// Collection and client models contain the same operation fields, with a
|
|
48
|
+
// different envelope. Keep the schema checked at the collection boundary.
|
|
49
|
+
const [name, model] = Object.entries(operation)[0];
|
|
50
|
+
this.operations.push({ ...model, name, namespace: collection.namespace });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
|
|
55
|
+
if (!this.supportsClientBulkWrite) {
|
|
56
|
+
this.fallback.push(() => collection.insertOne(document, { session: this.session }));
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
insertMany<T extends mongo.Document>(
|
|
63
|
+
collection: mongo.Collection<T>,
|
|
64
|
+
documents: mongo.OptionalUnlessRequiredId<T>[]
|
|
65
|
+
) {
|
|
66
|
+
if (documents.length === 0) return;
|
|
67
|
+
if (!this.supportsClientBulkWrite) {
|
|
68
|
+
this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
for (const document of documents) {
|
|
72
|
+
this.addModel<T>({
|
|
73
|
+
name: 'insertOne',
|
|
74
|
+
namespace: collection.namespace,
|
|
75
|
+
document: document as mongo.OptionalId<T>
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
deleteMany<T extends mongo.Document>(
|
|
81
|
+
collection: mongo.Collection<T>,
|
|
82
|
+
filter: mongo.Filter<T>,
|
|
83
|
+
checkDeletedCount?: (count: number) => void
|
|
84
|
+
) {
|
|
85
|
+
if (checkDeletedCount) {
|
|
86
|
+
// Client-bulk results are checked after all writes execute. A failed
|
|
87
|
+
// invariant must roll back the entire batch, not leave later writes visible.
|
|
88
|
+
if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
|
|
89
|
+
}
|
|
90
|
+
if (!this.supportsClientBulkWrite) {
|
|
91
|
+
this.fallback.push(async () => {
|
|
92
|
+
const result = await collection.deleteMany(filter, { session: this.session });
|
|
93
|
+
checkDeletedCount?.(result.deletedCount);
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (checkDeletedCount) {
|
|
98
|
+
this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
|
|
99
|
+
}
|
|
100
|
+
this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
updateOne<T extends mongo.Document>(
|
|
104
|
+
collection: mongo.Collection<T>,
|
|
105
|
+
filter: mongo.Filter<T>,
|
|
106
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
107
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
108
|
+
) {
|
|
109
|
+
if (!this.supportsClientBulkWrite) {
|
|
110
|
+
this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
updateMany<T extends mongo.Document>(
|
|
117
|
+
collection: mongo.Collection<T>,
|
|
118
|
+
filter: mongo.Filter<T>,
|
|
119
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
120
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
121
|
+
) {
|
|
122
|
+
if (!this.supportsClientBulkWrite) {
|
|
123
|
+
this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
|
|
130
|
+
// The collection-specific methods validate the schema before erasing it here.
|
|
131
|
+
this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Execute the queued writes.
|
|
136
|
+
*
|
|
137
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
|
|
138
|
+
* The command may be split if there are a large number of operations.
|
|
139
|
+
*
|
|
140
|
+
* On earlier MongoDB versions, this executes each queued operation individually.
|
|
141
|
+
*
|
|
142
|
+
* If no operations were queued, this is a no-op.
|
|
143
|
+
*/
|
|
144
|
+
async execute(): Promise<void> {
|
|
145
|
+
if (!this.supportsClientBulkWrite) {
|
|
146
|
+
// Sessions cannot execute concurrent operations within a transaction.
|
|
147
|
+
for (const write of this.fallback) await write();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (this.operations.length === 0) return;
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
const result = await this.client.bulkWrite(this.operations, {
|
|
154
|
+
session: this.session,
|
|
155
|
+
ordered: this.ordered,
|
|
156
|
+
verboseResults: this.deleteChecks.length > 0
|
|
157
|
+
});
|
|
158
|
+
for (const { index, check } of this.deleteChecks) {
|
|
159
|
+
const deleted = result.deleteResults?.get(index);
|
|
160
|
+
if (deleted == null) throw new Error('Missing client bulk delete result');
|
|
161
|
+
check(deleted.deletedCount);
|
|
162
|
+
}
|
|
163
|
+
} catch (error) {
|
|
164
|
+
// The driver wraps command failures without copying their error labels.
|
|
165
|
+
// Preserve withTransaction's retry handling for e.g. write conflicts.
|
|
166
|
+
if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
|
|
167
|
+
throw error.cause;
|
|
168
|
+
}
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
@@ -1,29 +1,31 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
1
|
import * as framework from '@powersync/lib-services-framework';
|
|
3
|
-
import { GetCheckpointChangesOptions,
|
|
2
|
+
import { GetCheckpointChangesOptions, storage } from '@powersync/service-core';
|
|
4
3
|
import { VersionedPowerSyncMongo } from './db.js';
|
|
5
4
|
|
|
6
5
|
export type MongoCheckpointAPIOptions = {
|
|
7
6
|
db: VersionedPowerSyncMongo;
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
writeCheckpointMode: storage.WriteCheckpointModeConfig;
|
|
8
|
+
replicationStreamId: number;
|
|
10
9
|
};
|
|
11
10
|
|
|
12
11
|
export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
13
12
|
readonly db: VersionedPowerSyncMongo;
|
|
13
|
+
readonly replicationStreamId: number;
|
|
14
|
+
|
|
14
15
|
private _mode: storage.WriteCheckpointMode;
|
|
15
16
|
|
|
16
17
|
constructor(options: MongoCheckpointAPIOptions) {
|
|
17
18
|
this.db = options.db;
|
|
18
|
-
this._mode = options.mode;
|
|
19
|
+
this._mode = options.writeCheckpointMode.mode;
|
|
20
|
+
this.replicationStreamId = options.replicationStreamId;
|
|
19
21
|
}
|
|
20
22
|
|
|
21
23
|
get writeCheckpointMode() {
|
|
22
24
|
return this._mode;
|
|
23
25
|
}
|
|
24
26
|
|
|
25
|
-
setWriteCheckpointMode(
|
|
26
|
-
this._mode = mode;
|
|
27
|
+
setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
|
|
28
|
+
this._mode = config.mode;
|
|
27
29
|
}
|
|
28
30
|
|
|
29
31
|
async createManagedWriteCheckpoints(
|
|
@@ -93,7 +95,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
93
95
|
},
|
|
94
96
|
upsert: true
|
|
95
97
|
}
|
|
96
|
-
}))
|
|
98
|
+
})),
|
|
99
|
+
{ ordered: false }
|
|
97
100
|
);
|
|
98
101
|
|
|
99
102
|
const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
|
|
@@ -178,7 +181,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
178
181
|
upsert: true
|
|
179
182
|
}
|
|
180
183
|
};
|
|
181
|
-
})
|
|
184
|
+
}),
|
|
185
|
+
{ ordered: false }
|
|
182
186
|
);
|
|
183
187
|
|
|
184
188
|
// Fetch the final ids separately so stale requests can still return the
|
|
@@ -247,6 +251,7 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
247
251
|
}
|
|
248
252
|
|
|
249
253
|
protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) {
|
|
254
|
+
// Legacy V1/V2 implementation.
|
|
250
255
|
const { user_id, sync_rules_id } = filters;
|
|
251
256
|
const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
|
|
252
257
|
user_id,
|
|
@@ -301,8 +306,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
301
306
|
};
|
|
302
307
|
}
|
|
303
308
|
|
|
304
|
-
|
|
309
|
+
protected async getCustomWriteCheckpointChanges(options: GetCheckpointChangesOptions) {
|
|
305
310
|
const limit = 1000;
|
|
311
|
+
// Legacy V1/V2 implementation.
|
|
306
312
|
const changes = await this.db.custom_write_checkpoints
|
|
307
313
|
.find(
|
|
308
314
|
{
|
|
@@ -332,39 +338,3 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
332
338
|
};
|
|
333
339
|
}
|
|
334
340
|
}
|
|
335
|
-
|
|
336
|
-
export async function batchCreateCustomWriteCheckpoints(
|
|
337
|
-
db: VersionedPowerSyncMongo,
|
|
338
|
-
session: mongo.ClientSession,
|
|
339
|
-
checkpoints: storage.CustomWriteCheckpointOptions[],
|
|
340
|
-
opId: InternalOpId
|
|
341
|
-
): Promise<void> {
|
|
342
|
-
if (checkpoints.length == 0) {
|
|
343
|
-
return;
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
await db.custom_write_checkpoints.bulkWrite(
|
|
347
|
-
checkpoints.map((checkpointOptions) => {
|
|
348
|
-
const set: Record<string, unknown> = {
|
|
349
|
-
checkpoint: checkpointOptions.checkpoint,
|
|
350
|
-
sync_rules_id: checkpointOptions.sync_rules_id,
|
|
351
|
-
op_id: opId
|
|
352
|
-
};
|
|
353
|
-
if (checkpointOptions.checkpoint_requested_at != null) {
|
|
354
|
-
set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
return {
|
|
358
|
-
updateOne: {
|
|
359
|
-
filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
|
|
360
|
-
update: {
|
|
361
|
-
$set: set,
|
|
362
|
-
...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
|
|
363
|
-
},
|
|
364
|
-
upsert: true
|
|
365
|
-
}
|
|
366
|
-
};
|
|
367
|
-
}),
|
|
368
|
-
{ session }
|
|
369
|
-
);
|
|
370
|
-
}
|
|
@@ -2,21 +2,28 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { storage } from '@powersync/service-core';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Update pipeline to
|
|
5
|
+
* Update pipeline to stop a deploying replication stream, covering all storage versions.
|
|
6
6
|
*
|
|
7
7
|
* Roughly equivalent to:
|
|
8
8
|
* $set: {
|
|
9
|
-
* state:
|
|
10
|
-
* 'sync_configs.$[].state':
|
|
9
|
+
* state: STOP,
|
|
10
|
+
* 'sync_configs.$[].state': STOP
|
|
11
11
|
* }
|
|
12
12
|
*
|
|
13
13
|
* The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
|
|
14
|
+
*
|
|
15
|
+
* This can be used for specific use cases:
|
|
16
|
+
* 1. Stop an existing PROCESSING stream, when it is being replaced by a new PROCESSING one.
|
|
17
|
+
* 2. Stop an existing ACTIVE or ERRORED stream, when it is being replaced by a new ACTIVE one.
|
|
18
|
+
*
|
|
19
|
+
* Do not use this when an ACTIVE stream transitions to ERRORED, and is being replaced by a new PROCESSING one.
|
|
20
|
+
* In that case, we need to maintain the ERRORED state, which this pipeline does not cover.
|
|
14
21
|
*/
|
|
15
|
-
export function
|
|
22
|
+
export function stopReplicationStreamPipeline(): mongo.Document[] {
|
|
16
23
|
return [
|
|
17
24
|
{
|
|
18
25
|
$set: {
|
|
19
|
-
state,
|
|
26
|
+
state: storage.SyncRuleState.STOP,
|
|
20
27
|
sync_configs: {
|
|
21
28
|
$cond: [
|
|
22
29
|
{ $isArray: '$sync_configs' },
|
|
@@ -25,7 +32,7 @@ export function syncRuleStateUpdatePipeline(state: storage.SyncRuleState): mongo
|
|
|
25
32
|
input: '$sync_configs',
|
|
26
33
|
as: 'config',
|
|
27
34
|
in: {
|
|
28
|
-
$mergeObjects: ['$$config', { state }]
|
|
35
|
+
$mergeObjects: ['$$config', { state: storage.SyncRuleState.STOP }]
|
|
29
36
|
}
|
|
30
37
|
}
|
|
31
38
|
},
|
|
@@ -1,9 +1,31 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { InternalOpId } from '@powersync/service-core';
|
|
2
|
+
import { InternalOpId, ReplicationCheckpoint } from '@powersync/service-core';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
|
|
5
|
-
export interface MongoSyncBucketStorageCheckpoint {
|
|
5
|
+
export interface MongoSyncBucketStorageCheckpoint extends ReplicationCheckpoint {
|
|
6
6
|
checkpoint: InternalOpId;
|
|
7
7
|
snapshotTime: bson.Timestamp;
|
|
8
8
|
clusterTime: mongo.ClusterTime;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The stream's `parameter_compaction.checkpoint_changes_invalid_before` boundary, read in the
|
|
12
|
+
* same snapshot as {@link checkpoint} and {@link snapshotTime}.
|
|
13
|
+
*
|
|
14
|
+
* Parameter compaction may have deleted parameter entries below this boundary, so checkpoint
|
|
15
|
+
* change detection cannot enumerate individual lookup changes from a checkpoint below it.
|
|
16
|
+
*
|
|
17
|
+
* 0n for streams that have never been compacted.
|
|
18
|
+
*/
|
|
19
|
+
parameterChangesInvalidBefore: InternalOpId;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* MongoDB-specific version of GetCheckpointChangesOptions.
|
|
24
|
+
*
|
|
25
|
+
* The next checkpoint carries the snapshot and invalidation boundary that change detection must
|
|
26
|
+
* be evaluated against. Only `checkpoint` is used from the previous one.
|
|
27
|
+
*/
|
|
28
|
+
export interface MongoGetCheckpointChangesOptions {
|
|
29
|
+
lastCheckpoint: ReplicationCheckpoint;
|
|
30
|
+
nextCheckpoint: MongoSyncBucketStorageCheckpoint;
|
|
9
31
|
}
|
|
@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
|
|
|
8
8
|
import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
|
|
9
9
|
import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
|
|
10
10
|
import { MongoIdSequence } from '../MongoIdSequence.js';
|
|
11
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
11
12
|
import type { VersionedPowerSyncMongo } from '../db.js';
|
|
12
13
|
import { TaggedBucketParameterDocument } from '../models.js';
|
|
13
14
|
import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
|
|
@@ -65,6 +66,11 @@ export interface PersistedBatchOptions {
|
|
|
65
66
|
logger?: Logger;
|
|
66
67
|
objectStorage?: ObjectStorage;
|
|
67
68
|
inlineThresholdBytes?: number;
|
|
69
|
+
/**
|
|
70
|
+
* Aborts in-flight object storage uploads when replication stops.
|
|
71
|
+
*/
|
|
72
|
+
signal?: AbortSignal;
|
|
73
|
+
objectStorageUsageWriterId?: string;
|
|
68
74
|
}
|
|
69
75
|
|
|
70
76
|
/**
|
|
@@ -81,6 +87,8 @@ export abstract class PersistedBatch {
|
|
|
81
87
|
|
|
82
88
|
protected readonly objectStorage?: ObjectStorage;
|
|
83
89
|
protected readonly inlineThresholdBytes: number = DEFAULT_INLINE_THRESHOLD_BYTES;
|
|
90
|
+
protected readonly signal?: AbortSignal;
|
|
91
|
+
protected readonly objectStorageUsageWriterId?: string;
|
|
84
92
|
|
|
85
93
|
/**
|
|
86
94
|
* For debug logging only.
|
|
@@ -102,6 +110,8 @@ export abstract class PersistedBatch {
|
|
|
102
110
|
this.currentSize = writtenSize;
|
|
103
111
|
this.logger = options?.logger ?? defaultLogger;
|
|
104
112
|
this.objectStorage = options?.objectStorage;
|
|
113
|
+
this.signal = options?.signal;
|
|
114
|
+
this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
|
|
105
115
|
if (options?.inlineThresholdBytes != null) {
|
|
106
116
|
this.inlineThresholdBytes = options.inlineThresholdBytes;
|
|
107
117
|
}
|
|
@@ -124,7 +134,8 @@ export abstract class PersistedBatch {
|
|
|
124
134
|
remaining_buckets.set(currentBucketKey(mapped), mapped);
|
|
125
135
|
}
|
|
126
136
|
|
|
127
|
-
const
|
|
137
|
+
const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
|
|
138
|
+
const dchecksum = BigInt(utils.hashDelete(subkey));
|
|
128
139
|
|
|
129
140
|
for (const evaluated of options.evaluated) {
|
|
130
141
|
const definitionId = this.getBucketDefinitionId(evaluated.source);
|
|
@@ -159,6 +170,7 @@ export abstract class PersistedBatch {
|
|
|
159
170
|
bucket: evaluated.bucket,
|
|
160
171
|
sourceTableId: options.table.id,
|
|
161
172
|
sourceKey: options.sourceKey,
|
|
173
|
+
subkey,
|
|
162
174
|
table: evaluated.table,
|
|
163
175
|
rowId: evaluated.id,
|
|
164
176
|
checksum: BigInt(checksum),
|
|
@@ -181,6 +193,7 @@ export abstract class PersistedBatch {
|
|
|
181
193
|
op_id,
|
|
182
194
|
sourceTableId: options.table.id,
|
|
183
195
|
sourceKey: options.sourceKey,
|
|
196
|
+
subkey,
|
|
184
197
|
table: bucket.table,
|
|
185
198
|
rowId: bucket.id,
|
|
186
199
|
checksum: dchecksum
|
|
@@ -204,19 +217,27 @@ export abstract class PersistedBatch {
|
|
|
204
217
|
|
|
205
218
|
protected abstract get currentDataCount(): number;
|
|
206
219
|
|
|
207
|
-
protected abstract
|
|
220
|
+
protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
|
|
208
221
|
|
|
209
|
-
protected abstract
|
|
222
|
+
protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
|
|
210
223
|
|
|
211
|
-
protected abstract
|
|
224
|
+
protected abstract queueCurrentData(writes: MongoWriteBatch): void;
|
|
212
225
|
|
|
213
|
-
protected abstract
|
|
226
|
+
protected abstract queueBucketStates(writes: MongoWriteBatch): void;
|
|
214
227
|
|
|
215
228
|
protected abstract resetCurrentData(): void;
|
|
216
229
|
|
|
217
230
|
protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
|
|
218
231
|
protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
|
|
219
232
|
|
|
233
|
+
/**
|
|
234
|
+
* New bucket operations persist protocol subkeys during replication. Older
|
|
235
|
+
* V1 documents may not have one, so sync reads retain a fallback.
|
|
236
|
+
*/
|
|
237
|
+
protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
|
|
238
|
+
return replicaIdToSubkey(table, key);
|
|
239
|
+
}
|
|
240
|
+
|
|
220
241
|
protected get bucketDataCount(): number {
|
|
221
242
|
return this.bucketData.length;
|
|
222
243
|
}
|
|
@@ -234,17 +255,45 @@ export abstract class PersistedBatch {
|
|
|
234
255
|
bucket,
|
|
235
256
|
lastOp: op_id,
|
|
236
257
|
incrementCount: 1,
|
|
237
|
-
incrementBytes: bytes
|
|
258
|
+
incrementBytes: bytes,
|
|
259
|
+
incrementChunks: 0
|
|
238
260
|
});
|
|
239
261
|
}
|
|
240
262
|
}
|
|
241
263
|
|
|
264
|
+
/**
|
|
265
|
+
* V3 persists operations in chunks. Keep this separate from incrementBucket:
|
|
266
|
+
* operation counts are known while evaluating rows, while chunk counts and
|
|
267
|
+
* exact persisted bytes are only known after the writer has chunked a flush.
|
|
268
|
+
*/
|
|
269
|
+
protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
|
|
270
|
+
const key = `${definitionId ?? ''}:${bucket}`;
|
|
271
|
+
const existingState = this.bucketStates.get(key);
|
|
272
|
+
if (existingState != null) {
|
|
273
|
+
existingState.incrementChunks += 1;
|
|
274
|
+
existingState.incrementBytes += bytes;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* V3's compactor calculates byte deltas from persisted chunk metadata.
|
|
280
|
+
* Replace the writer's per-operation estimate with those exact sizes before
|
|
281
|
+
* flushing the corresponding bucket state.
|
|
282
|
+
*/
|
|
283
|
+
protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
|
|
284
|
+
const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
|
|
285
|
+
if (state != null) {
|
|
286
|
+
state.incrementBytes = 0;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
242
290
|
protected addBucketDataPut(options: {
|
|
243
291
|
op_id: InternalOpId;
|
|
244
292
|
bucketKey: BucketKey;
|
|
245
293
|
bucket: string;
|
|
246
294
|
sourceTableId: storage.SourceTable['id'];
|
|
247
295
|
sourceKey: storage.ReplicaId;
|
|
296
|
+
subkey: string;
|
|
248
297
|
table: string;
|
|
249
298
|
rowId: string;
|
|
250
299
|
checksum: bigint;
|
|
@@ -256,6 +305,7 @@ export abstract class PersistedBatch {
|
|
|
256
305
|
op: 'PUT',
|
|
257
306
|
source_table: mongoTableId(options.sourceTableId),
|
|
258
307
|
source_key: options.sourceKey,
|
|
308
|
+
subkey: options.subkey,
|
|
259
309
|
table: options.table,
|
|
260
310
|
row_id: options.rowId,
|
|
261
311
|
checksum: options.checksum,
|
|
@@ -268,6 +318,7 @@ export abstract class PersistedBatch {
|
|
|
268
318
|
bucketKey: BucketKey;
|
|
269
319
|
sourceTableId: storage.SourceTable['id'];
|
|
270
320
|
sourceKey: storage.ReplicaId;
|
|
321
|
+
subkey: string;
|
|
271
322
|
table: string;
|
|
272
323
|
rowId: string;
|
|
273
324
|
checksum: bigint;
|
|
@@ -278,6 +329,7 @@ export abstract class PersistedBatch {
|
|
|
278
329
|
op: 'REMOVE',
|
|
279
330
|
source_table: mongoTableId(options.sourceTableId),
|
|
280
331
|
source_key: options.sourceKey,
|
|
332
|
+
subkey: options.subkey,
|
|
281
333
|
table: options.table,
|
|
282
334
|
row_id: options.rowId,
|
|
283
335
|
checksum: options.checksum,
|
|
@@ -297,56 +349,59 @@ export abstract class PersistedBatch {
|
|
|
297
349
|
async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
|
|
298
350
|
const startAt = performance.now();
|
|
299
351
|
let flushedSomething = false;
|
|
352
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
300
353
|
if (this.bucketDataCount > 0) {
|
|
301
354
|
flushedSomething = true;
|
|
302
|
-
await this.
|
|
355
|
+
await this.queueBucketData(writes);
|
|
303
356
|
}
|
|
304
357
|
if (this.bucketParameters.length > 0) {
|
|
305
358
|
flushedSomething = true;
|
|
306
|
-
|
|
359
|
+
this.queueBucketParameters(writes);
|
|
307
360
|
}
|
|
308
361
|
if (this.currentDataCount > 0) {
|
|
309
362
|
flushedSomething = true;
|
|
310
|
-
|
|
363
|
+
this.queueCurrentData(writes);
|
|
311
364
|
}
|
|
312
365
|
|
|
313
366
|
if (this.bucketStates.size > 0) {
|
|
314
367
|
flushedSomething = true;
|
|
315
|
-
|
|
368
|
+
this.queueBucketStates(writes);
|
|
316
369
|
}
|
|
317
370
|
|
|
371
|
+
await writes.execute();
|
|
372
|
+
|
|
318
373
|
if (flushedSomething) {
|
|
319
374
|
const duration = Math.round(performance.now() - startAt);
|
|
320
375
|
if (options?.oldestUncommittedChange != null) {
|
|
321
376
|
const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
|
|
322
377
|
|
|
323
378
|
this.logger.info(
|
|
324
|
-
`Flushed ${this.bucketDataCount}
|
|
379
|
+
`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
|
|
325
380
|
this.currentDataCount
|
|
326
|
-
}
|
|
381
|
+
} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
|
|
327
382
|
{
|
|
328
383
|
flushed: {
|
|
329
384
|
duration: duration,
|
|
330
385
|
size: this.currentSize,
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
386
|
+
bucket_ops_count: this.bucketDataCount,
|
|
387
|
+
parameter_indexes_count: this.bucketParameters.length,
|
|
388
|
+
source_records_count: this.currentDataCount,
|
|
334
389
|
replication_lag_seconds: replicationLag
|
|
335
390
|
}
|
|
336
391
|
}
|
|
337
392
|
);
|
|
338
393
|
} else {
|
|
339
394
|
this.logger.info(
|
|
340
|
-
`Flushed ${this.bucketDataCount}
|
|
395
|
+
`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
|
|
341
396
|
this.currentDataCount
|
|
342
|
-
}
|
|
397
|
+
} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
|
|
343
398
|
{
|
|
344
399
|
flushed: {
|
|
345
400
|
duration: duration,
|
|
346
401
|
size: this.currentSize,
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
402
|
+
bucket_ops_count: this.bucketDataCount,
|
|
403
|
+
parameter_indexes_count: this.bucketParameters.length,
|
|
404
|
+
source_records_count: this.currentDataCount
|
|
350
405
|
}
|
|
351
406
|
}
|
|
352
407
|
);
|
|
@@ -377,4 +432,5 @@ export interface BucketStateUpdate {
|
|
|
377
432
|
lastOp: InternalOpId;
|
|
378
433
|
incrementCount: number;
|
|
379
434
|
incrementBytes: number;
|
|
435
|
+
incrementChunks: number;
|
|
380
436
|
}
|
|
@@ -58,6 +58,10 @@ export abstract class BaseVersionedPowerSyncMongo {
|
|
|
58
58
|
return this.upstream.notifyCheckpoint();
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }) {
|
|
62
|
+
return this.upstream.createWriteBatch(session, options);
|
|
63
|
+
}
|
|
64
|
+
|
|
61
65
|
protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
|
|
62
66
|
return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
|
|
63
67
|
}
|