@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { addChecksums, utils } from '@powersync/service-core';
|
|
3
4
|
import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
|
|
4
5
|
import { MongoCompactor } from '../MongoCompactor.js';
|
|
6
|
+
import { cacheKey } from '../OperationBatch.js';
|
|
5
7
|
import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
|
|
6
8
|
export class MongoCompactorV1 extends MongoCompactor {
|
|
9
|
+
updates = [];
|
|
7
10
|
async *dirtyBucketBatches(options) {
|
|
8
11
|
if (options.minBucketChanges <= 0) {
|
|
9
12
|
throw new ReplicationAssertionError('minBucketChanges must be >= 1');
|
|
@@ -31,7 +34,7 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
31
34
|
return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
|
|
32
35
|
bucket,
|
|
33
36
|
end: this.maxOpId
|
|
34
|
-
})));
|
|
37
|
+
})), { readOptions: { readConcern: 'snapshot' } });
|
|
35
38
|
}
|
|
36
39
|
bucketStateFilter(bucket, _definitionId) {
|
|
37
40
|
return {
|
|
@@ -41,12 +44,258 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
41
44
|
}
|
|
42
45
|
};
|
|
43
46
|
}
|
|
44
|
-
|
|
47
|
+
getBucketDataContext(bucket) {
|
|
45
48
|
return new SingleBucketStoreV1(this.db, {
|
|
46
49
|
replicationStreamId: this.group_id,
|
|
47
50
|
definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
|
|
48
51
|
bucket
|
|
49
52
|
});
|
|
50
53
|
}
|
|
54
|
+
async compactSingleBucket(bucket) {
|
|
55
|
+
// Do not carry queued writes from a failed attempt into the rescan
|
|
56
|
+
this.updates = [];
|
|
57
|
+
const idLimitBytes = this.idLimitBytes;
|
|
58
|
+
const bucketContext = this.getBucketDataContext(bucket);
|
|
59
|
+
const currentState = {
|
|
60
|
+
bucket,
|
|
61
|
+
definitionId: bucketContext.key.definitionId,
|
|
62
|
+
seen: new Map(),
|
|
63
|
+
trackingSize: 0,
|
|
64
|
+
lastNotPut: null,
|
|
65
|
+
opsSincePut: 0,
|
|
66
|
+
checksum: 0,
|
|
67
|
+
opCount: 0,
|
|
68
|
+
opBytes: 0
|
|
69
|
+
};
|
|
70
|
+
// Constant lower bound.
|
|
71
|
+
const lowerBound = bucketContext.minId;
|
|
72
|
+
// Upper bound is adjusted for each batch.
|
|
73
|
+
let upperBound = bucketContext.maxId;
|
|
74
|
+
// Paginate through bucket data in batches to avoid cursor timeouts.
|
|
75
|
+
while (true) {
|
|
76
|
+
this.signal?.throwIfAborted();
|
|
77
|
+
// Query one batch at a time, to avoid cursor timeouts.
|
|
78
|
+
const pipeline = [
|
|
79
|
+
{
|
|
80
|
+
$match: {
|
|
81
|
+
_id: {
|
|
82
|
+
$gte: lowerBound,
|
|
83
|
+
$lt: upperBound
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
{ $sort: { _id: -1 } },
|
|
88
|
+
{ $limit: this.moveBatchQueryLimit },
|
|
89
|
+
{
|
|
90
|
+
$project: {
|
|
91
|
+
_id: 1,
|
|
92
|
+
op: 1,
|
|
93
|
+
table: 1,
|
|
94
|
+
row_id: 1,
|
|
95
|
+
source_table: 1,
|
|
96
|
+
source_key: 1,
|
|
97
|
+
checksum: 1,
|
|
98
|
+
size: { $bsonSize: '$$ROOT' }
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
];
|
|
102
|
+
const cursor = bucketContext.collection.aggregate(pipeline, {
|
|
103
|
+
// batchSize is 1 more than limit to auto-close the cursor.
|
|
104
|
+
// See https://github.com/mongodb/node-mongodb-native/pull/4580
|
|
105
|
+
batchSize: this.moveBatchQueryLimit + 1
|
|
106
|
+
});
|
|
107
|
+
// We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
|
|
108
|
+
// Instead, we load up to the limit.
|
|
109
|
+
const rawBatch = await cursor.toArray();
|
|
110
|
+
const batch = rawBatch.map((document) => {
|
|
111
|
+
const { size, ...rest } = document;
|
|
112
|
+
return {
|
|
113
|
+
doc: bucketContext.fromPersistedDocument(rest),
|
|
114
|
+
size
|
|
115
|
+
};
|
|
116
|
+
});
|
|
117
|
+
if (batch.length == 0) {
|
|
118
|
+
// We've reached the end.
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
// Reuse the exact collection _id value from Mongo for the next bound.
|
|
122
|
+
upperBound = rawBatch[rawBatch.length - 1]._id;
|
|
123
|
+
for (const { doc, size } of batch) {
|
|
124
|
+
if (doc.o > this.maxOpId) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
|
|
128
|
+
currentState.opCount += 1;
|
|
129
|
+
let isPersistentPut = doc.op == 'PUT';
|
|
130
|
+
currentState.opBytes += Number(size);
|
|
131
|
+
if (doc.op == 'REMOVE' || doc.op == 'PUT') {
|
|
132
|
+
const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table, doc.source_key)}`;
|
|
133
|
+
const targetOp = currentState.seen.get(key);
|
|
134
|
+
if (targetOp) {
|
|
135
|
+
// Will convert to MOVE, so don't count as PUT.
|
|
136
|
+
isPersistentPut = false;
|
|
137
|
+
this.updates.push({
|
|
138
|
+
updateOne: {
|
|
139
|
+
filter: { _id: bucketContext.docId(doc.o) },
|
|
140
|
+
update: {
|
|
141
|
+
$set: {
|
|
142
|
+
op: 'MOVE',
|
|
143
|
+
target_op: targetOp
|
|
144
|
+
},
|
|
145
|
+
$unset: {
|
|
146
|
+
source_table: 1,
|
|
147
|
+
source_key: 1,
|
|
148
|
+
table: 1,
|
|
149
|
+
row_id: 1,
|
|
150
|
+
data: 1
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
// TODO: better estimate for this.
|
|
156
|
+
currentState.opBytes += 200 - Number(size);
|
|
157
|
+
}
|
|
158
|
+
else if (currentState.trackingSize < idLimitBytes) {
|
|
159
|
+
// flatstr reduces the memory usage by flattening the string.
|
|
160
|
+
currentState.seen.set(utils.flatstr(key), doc.o);
|
|
161
|
+
// length + 16 for the string
|
|
162
|
+
// 24 for the bigint
|
|
163
|
+
// 50 for map overhead
|
|
164
|
+
// 50 for additional overhead
|
|
165
|
+
currentState.trackingSize += key.length + 140;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (isPersistentPut) {
|
|
169
|
+
currentState.lastNotPut = null;
|
|
170
|
+
currentState.opsSincePut = 0;
|
|
171
|
+
}
|
|
172
|
+
else if (doc.op != 'CLEAR') {
|
|
173
|
+
if (currentState.lastNotPut == null) {
|
|
174
|
+
currentState.lastNotPut = doc.o;
|
|
175
|
+
}
|
|
176
|
+
currentState.opsSincePut += 1;
|
|
177
|
+
}
|
|
178
|
+
if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
|
|
179
|
+
await this.flush(bucketContext);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
|
|
183
|
+
}
|
|
184
|
+
// Free memory before clearing the bucket.
|
|
185
|
+
currentState.seen.clear();
|
|
186
|
+
if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
|
|
187
|
+
this.logger.info(`Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
|
|
188
|
+
// Need flush() before clear().
|
|
189
|
+
await this.flush(bucketContext);
|
|
190
|
+
await this.clearBucket(currentState, bucketContext);
|
|
191
|
+
}
|
|
192
|
+
// Do this after clearBucket so we have accurate counts.
|
|
193
|
+
this.updateBucketChecksums(currentState, this.maxOpId);
|
|
194
|
+
// Need another flush after updateBucketChecksums().
|
|
195
|
+
await this.flush(bucketContext);
|
|
196
|
+
}
|
|
197
|
+
async flush(bucketContext) {
|
|
198
|
+
if (this.updates.length > 0) {
|
|
199
|
+
this.logger.info(`Compacting ${this.updates.length} ops`);
|
|
200
|
+
await bucketContext.collection.bulkWrite(this.updates, {
|
|
201
|
+
// Order is not important. Since checksums are not affected, these operations can happen in any order,
|
|
202
|
+
// and it's fine if the operations are partially applied. Each individual operation is atomic.
|
|
203
|
+
ordered: false
|
|
204
|
+
});
|
|
205
|
+
this.updates = [];
|
|
206
|
+
}
|
|
207
|
+
await this.flushBucketStateUpdates();
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Perform a CLEAR compact for a bucket.
|
|
211
|
+
*
|
|
212
|
+
* @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
|
|
213
|
+
*/
|
|
214
|
+
async clearBucket(currentState, bucketContext) {
|
|
215
|
+
const clearOp = currentState.lastNotPut;
|
|
216
|
+
const opFilter = {
|
|
217
|
+
_id: {
|
|
218
|
+
$gte: bucketContext.minId,
|
|
219
|
+
$lte: bucketContext.docId(clearOp)
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
const session = this.db.client.startSession();
|
|
223
|
+
try {
|
|
224
|
+
let done = false;
|
|
225
|
+
while (!done) {
|
|
226
|
+
this.signal?.throwIfAborted();
|
|
227
|
+
let opCountDiff = 0;
|
|
228
|
+
// Do the CLEAR operation in batches, with each batch a separate transaction.
|
|
229
|
+
// The state after each batch is fully consistent.
|
|
230
|
+
// We need a transaction per batch to make sure checksums stay consistent.
|
|
231
|
+
await session.withTransaction(async () => {
|
|
232
|
+
const query = bucketContext.collection.find(opFilter, {
|
|
233
|
+
session,
|
|
234
|
+
sort: { _id: 1 },
|
|
235
|
+
projection: {
|
|
236
|
+
_id: 1,
|
|
237
|
+
op: 1,
|
|
238
|
+
checksum: 1,
|
|
239
|
+
target_op: 1
|
|
240
|
+
},
|
|
241
|
+
limit: this.clearBatchLimit
|
|
242
|
+
});
|
|
243
|
+
let checksum = 0;
|
|
244
|
+
let lastOp = null;
|
|
245
|
+
let targetOp = null;
|
|
246
|
+
let gotAnOp = false;
|
|
247
|
+
let numberOfOpsToClear = 0;
|
|
248
|
+
for await (const rawOp of query.stream()) {
|
|
249
|
+
const op = bucketContext.fromPartialPersistedDocument(rawOp);
|
|
250
|
+
if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
|
|
251
|
+
checksum = utils.addChecksums(checksum, Number(op.checksum));
|
|
252
|
+
lastOp = op;
|
|
253
|
+
numberOfOpsToClear += 1;
|
|
254
|
+
if (op.op != 'CLEAR') {
|
|
255
|
+
gotAnOp = true;
|
|
256
|
+
}
|
|
257
|
+
if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
|
|
258
|
+
targetOp = op.target_op;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
else {
|
|
262
|
+
throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
if (!gotAnOp) {
|
|
266
|
+
done = true;
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
|
|
270
|
+
await bucketContext.collection.deleteMany({
|
|
271
|
+
_id: {
|
|
272
|
+
$gte: bucketContext.minId,
|
|
273
|
+
$lte: bucketContext.docId(lastOp.o)
|
|
274
|
+
}
|
|
275
|
+
}, { session });
|
|
276
|
+
const op = bucketContext.toPersistedDocument({
|
|
277
|
+
o: lastOp.o,
|
|
278
|
+
op: 'CLEAR',
|
|
279
|
+
checksum: BigInt(checksum),
|
|
280
|
+
data: null,
|
|
281
|
+
target_op: targetOp
|
|
282
|
+
});
|
|
283
|
+
await bucketContext.collection.insertOne(op, { session });
|
|
284
|
+
opCountDiff = -numberOfOpsToClear + 1;
|
|
285
|
+
}, {
|
|
286
|
+
writeConcern: { w: 'majority' },
|
|
287
|
+
readConcern: { level: 'snapshot' }
|
|
288
|
+
});
|
|
289
|
+
// Update outside the transaction, since the transaction can be retried multiple times.
|
|
290
|
+
currentState.opCount += opCountDiff;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
finally {
|
|
294
|
+
await session.endSession();
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
formatBucketDataKey(doc) {
|
|
298
|
+
return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
|
|
299
|
+
}
|
|
51
300
|
}
|
|
52
301
|
//# sourceMappingURL=MongoCompactorV1.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAW,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGvE,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAmC,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACvF,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAK/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK1C,OAAO,GAAwD,EAAE,CAAC;IAEnE,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,EACH,EAAE,WAAW,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,CAC7C,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,MAAc;QACzC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAChD,mEAAmE;QACnE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,YAAY,GAAuB;YACvC,MAAM;YACN,YAAY,EAAE,aAAa,CAAC,GAAG,CAAC,YAAY;YAC5C,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,CAAC;QAEF,wBAAwB;QACxB,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QACvC,0CAA0C;QAC1C,IAAI,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QAErC,oEAAoE;QACpE,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,uDAAuD;YACvD,MAAM,QAAQ,GAAG;gBACf;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,SAAS,CAAmD,QAAQ,EAAE;gBAC5G,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,mBAAmB,GAAG,CAAC;aACxC,CAAC,CAAC;YACH,+GAA+G;YAC/G,oCAAoC;YACpC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACtC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;gBACnC,OAAO;oBACL,GAAG,EAAE,aAAa,CAAC,qBAAqB,CAAC,IAAI,CAAC;oBAC9C,IAAI;iBACL,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,yBAAyB;gBACzB,MAAM;YACR,CAAC;YAED,sEAAsE;YACtE,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAE/C,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,+CAA+C;wBAC/C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gCAC3C,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACiD;6BACrD;yBACF,CAAC,CAAC;wBAEH,kCAAkC;wBAClC,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC7C,CAAC;yBAAM,IAAI,YAAY,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC;wBACpD,6DAA6D;wBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;wBACjD,6BAA6B;wBAC7B,oBAAoB;wBACpB,sBAAsB;wBACtB,6BAA6B;wBAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;oBAChD,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC;oBAClC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,MAAM,oBAAoB,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,0CAA0C;QAC1C,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sBAAsB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CAC5H,CAAC;YACF,+BAA+B;YAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACtD,CAAC;QAED,wDAAwD;QACxD,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,oDAAoD;QACpD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,aAAkC;QACpD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YAC1D,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBACrD,sGAAsG;gBACtG,8FAA8F;gBAC9F,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC,EAAE,aAAkC;QAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE,aAAa,CAAC,KAAK;gBACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;aACnC;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;gBAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CACzC,QAAQ,EACR;wBACE,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CACF,CAAC;oBACF,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,MAAM,GAA6D,IAAI,CAAC;oBAC5E,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACzC,MAAM,EAAE,GAAG,aAAa,CAAC,4BAA4B,CAAC,KAAK,CAAC,CAAC;wBAE7D,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,MAAM,GAAG,EAAE,CAAC;4BACZ,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;gCAC1E,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;4BAC1B,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1G,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;oBACjF,MAAM,aAAa,CAAC,UAAU,CAAC,UAAU,CACvC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE,aAAa,CAAC,KAAK;4BACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC;yBACrC;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;wBAC3C,CAAC,EAAE,MAAO,CAAC,CAAC;wBACZ,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,CAAC,CAAC;oBACH,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;oBAE1D,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,uFAAuF;gBACvF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,GAA2C;QACrE,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,mBAAmB,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;IACjF,CAAC;CACF"}
|
|
@@ -1,41 +1,40 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage
|
|
2
|
+
import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
|
|
3
3
|
import { ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
|
|
4
4
|
import { MongoBucketStorage } from '../../MongoBucketStorage.js';
|
|
5
|
-
import { MongoSyncBucketStorageCheckpoint
|
|
6
|
-
import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
5
|
+
import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
|
|
7
6
|
import { MongoChecksums } from '../MongoChecksums.js';
|
|
8
7
|
import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
|
|
9
8
|
import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
|
|
10
|
-
import {
|
|
9
|
+
import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
|
|
11
10
|
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
|
|
12
11
|
import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
|
|
13
12
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
13
|
+
export interface MongoSyncBucketStorageContextV1 {
|
|
14
|
+
db: VersionedPowerSyncMongoV1;
|
|
15
|
+
replicationStreamId: number;
|
|
16
|
+
readPreference?: mongo.ReadPreference;
|
|
17
|
+
}
|
|
14
18
|
export declare class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
15
19
|
readonly db: VersionedPowerSyncMongoV1;
|
|
16
20
|
readonly checksums: MongoChecksumsV1;
|
|
17
|
-
constructor(factory: MongoBucketStorage,
|
|
21
|
+
constructor(factory: MongoBucketStorage, replicationStreamId: number, replicationStream: MongoPersistedReplicationStream, replicationStreamName: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
|
|
18
22
|
protected initializeVersionStorage(): Promise<void>;
|
|
19
|
-
protected
|
|
23
|
+
protected fetchPersistedOpHead(): Promise<InternalOpId | null>;
|
|
24
|
+
protected createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
|
|
20
25
|
protected fetchCheckpointState(session: mongo.ClientSession): Promise<{
|
|
21
26
|
checkpoint: bigint;
|
|
22
27
|
lsn: string | null;
|
|
23
28
|
} | null>;
|
|
24
|
-
protected getWriterSyncState(): Promise<{
|
|
25
|
-
lastCheckpointLsn: string | null;
|
|
26
|
-
resumeFromLsn: string | null;
|
|
27
|
-
keepaliveOp: bigint | null;
|
|
28
|
-
syncConfigId: null;
|
|
29
|
-
}>;
|
|
30
29
|
protected terminateSyncRuleState(): Promise<void>;
|
|
31
|
-
protected getStatusImpl(): Promise<storage.
|
|
30
|
+
protected getStatusImpl(): Promise<storage.ReplicationStreamStatus>;
|
|
32
31
|
protected clearSyncRuleState(): Promise<void>;
|
|
33
32
|
protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
|
|
34
33
|
createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
|
|
35
34
|
protected createMongoParameterCompactor(checkpoint: InternalOpId, options: storage.CompactOptions): MongoParameterCompactor;
|
|
36
|
-
protected get versionContext():
|
|
35
|
+
protected get versionContext(): MongoSyncBucketStorageContextV1;
|
|
37
36
|
protected getParameterSetsImpl(checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
|
|
38
|
-
protected getBucketDataBatchImpl(checkpoint:
|
|
37
|
+
protected getBucketDataBatchImpl(checkpoint: MongoSyncBucketStorageCheckpoint, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
|
|
39
38
|
protected clearBucketData(signal?: AbortSignal): Promise<void>;
|
|
40
39
|
protected clearParameterIndexes(signal?: AbortSignal): Promise<void>;
|
|
41
40
|
protected clearSourceRecords(signal?: AbortSignal): Promise<void>;
|
|
@@ -44,7 +43,7 @@ export declare class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
44
43
|
protected getDataBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
|
|
45
44
|
protected getParameterBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
|
|
46
45
|
}
|
|
47
|
-
export declare function getParameterSetsV1(ctx:
|
|
48
|
-
export declare function getBucketDataBatchV1(ctx:
|
|
49
|
-
export declare function getDataBucketChangesV1(ctx:
|
|
50
|
-
export declare function getParameterBucketChangesV1(ctx:
|
|
46
|
+
export declare function getParameterSetsV1(ctx: MongoSyncBucketStorageContextV1, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
|
|
47
|
+
export declare function getBucketDataBatchV1(ctx: MongoSyncBucketStorageContextV1, checkpoint: MongoSyncBucketStorageCheckpoint, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
|
|
48
|
+
export declare function getDataBucketChangesV1(ctx: MongoSyncBucketStorageContextV1, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
|
|
49
|
+
export declare function getParameterBucketChangesV1(ctx: MongoSyncBucketStorageContextV1, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
|
|
@@ -3,7 +3,7 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
|
3
3
|
import { deserializeParameterLookup, internalToExternalOpId, maxLsn, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
|
|
4
4
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
5
5
|
import * as bson from 'bson';
|
|
6
|
-
import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
|
|
6
|
+
import { clearCollectionInIdBatches, clearCollectionInIdRanges, idPrefixFilter, mapOpEntry, readSingleBatch, retryOnMongoMaxTimeMSExpired, setSessionSnapshotTime } from '../../../utils/util.js';
|
|
7
7
|
import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js';
|
|
8
8
|
import { loadBucketDataDocumentV1 } from './models.js';
|
|
9
9
|
import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js';
|
|
@@ -11,15 +11,34 @@ import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
|
|
|
11
11
|
import { MongoCompactorV1 } from './MongoCompactorV1.js';
|
|
12
12
|
import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js';
|
|
13
13
|
export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
14
|
-
constructor(factory,
|
|
15
|
-
super(factory,
|
|
14
|
+
constructor(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
|
|
15
|
+
super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
16
16
|
}
|
|
17
17
|
async initializeVersionStorage() { }
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
async fetchPersistedOpHead() {
|
|
19
|
+
const doc = (await this.db.sync_rules.findOne({ _id: this.replicationStreamId }, { projection: { keepalive_op: 1, last_checkpoint: 1 } }));
|
|
20
|
+
// keepalive_op covers ops not yet in a checkpoint (cleared once checkpointed),
|
|
21
|
+
// so the head is the max of the two.
|
|
22
|
+
const keepaliveOp = doc?.keepalive_op == null ? null : BigInt(doc.keepalive_op);
|
|
23
|
+
const lastCheckpoint = doc?.last_checkpoint ?? null;
|
|
24
|
+
if (keepaliveOp == null && lastCheckpoint == null) {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint;
|
|
28
|
+
}
|
|
29
|
+
async createWriterImpl(options) {
|
|
30
|
+
const doc = (await this.db.sync_rules.findOne({
|
|
31
|
+
_id: this.replicationStreamId
|
|
32
|
+
}, { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }));
|
|
33
|
+
return new MongoBucketBatchV1({
|
|
34
|
+
...this.writerBatchOptions(options),
|
|
35
|
+
// Resume from the last consistent checkpoint, or the in-progress snapshot position if it is newer.
|
|
36
|
+
resumeFromLsn: maxLsn(doc?.last_checkpoint_lsn, doc?.snapshot_lsn),
|
|
37
|
+
keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null
|
|
38
|
+
});
|
|
20
39
|
}
|
|
21
40
|
async fetchCheckpointState(session) {
|
|
22
|
-
const doc = (await this.db.sync_rules.findOne({ _id: this.
|
|
41
|
+
const doc = (await this.db.sync_rules.findOne({ _id: this.replicationStreamId }, {
|
|
23
42
|
session,
|
|
24
43
|
projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
|
|
25
44
|
}));
|
|
@@ -31,21 +50,9 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
31
50
|
lsn: doc.last_checkpoint_lsn ?? null
|
|
32
51
|
};
|
|
33
52
|
}
|
|
34
|
-
async getWriterSyncState() {
|
|
35
|
-
const doc = (await this.db.sync_rules.findOne({
|
|
36
|
-
_id: this.group_id
|
|
37
|
-
}, { projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }));
|
|
38
|
-
const checkpointLsn = doc?.last_checkpoint_lsn ?? null;
|
|
39
|
-
return {
|
|
40
|
-
lastCheckpointLsn: checkpointLsn,
|
|
41
|
-
resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
|
|
42
|
-
keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
|
|
43
|
-
syncConfigId: null
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
53
|
async terminateSyncRuleState() {
|
|
47
54
|
await this.db.sync_rules.updateOne({
|
|
48
|
-
_id: this.
|
|
55
|
+
_id: this.replicationStreamId
|
|
49
56
|
}, {
|
|
50
57
|
$set: {
|
|
51
58
|
state: storage.SyncRuleState.TERMINATED,
|
|
@@ -56,30 +63,25 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
56
63
|
}
|
|
57
64
|
async getStatusImpl() {
|
|
58
65
|
const doc = (await this.db.sync_rules.findOne({
|
|
59
|
-
_id: this.
|
|
66
|
+
_id: this.replicationStreamId
|
|
60
67
|
}, {
|
|
61
68
|
projection: {
|
|
62
69
|
snapshot_done: 1,
|
|
63
70
|
last_checkpoint_lsn: 1,
|
|
64
|
-
|
|
65
|
-
snapshot_lsn: 1,
|
|
66
|
-
keepalive_op: 1
|
|
71
|
+
snapshot_lsn: 1
|
|
67
72
|
}
|
|
68
73
|
}));
|
|
69
74
|
if (doc == null) {
|
|
70
75
|
throw new ServiceAssertionError('Cannot find replication stream status');
|
|
71
76
|
}
|
|
72
77
|
return {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
active: doc.state == storage.SyncRuleState.ACTIVE,
|
|
76
|
-
checkpoint_lsn: doc.last_checkpoint_lsn,
|
|
77
|
-
keepalive_op: doc.keepalive_op == null ? null : BigInt(doc.keepalive_op)
|
|
78
|
+
snapshotDone: doc.snapshot_done && doc.last_checkpoint_lsn != null,
|
|
79
|
+
resumeLsn: maxLsn(doc.snapshot_lsn, doc.last_checkpoint_lsn)
|
|
78
80
|
};
|
|
79
81
|
}
|
|
80
82
|
async clearSyncRuleState() {
|
|
81
83
|
await this.db.sync_rules.updateOne({
|
|
82
|
-
_id: this.
|
|
84
|
+
_id: this.replicationStreamId
|
|
83
85
|
}, {
|
|
84
86
|
$set: {
|
|
85
87
|
snapshot_done: false,
|
|
@@ -94,23 +96,23 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
94
96
|
}, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS });
|
|
95
97
|
}
|
|
96
98
|
createMongoChecksums(options) {
|
|
97
|
-
return new MongoChecksumsV1(this.db, this.
|
|
99
|
+
return new MongoChecksumsV1(this.db, this.replicationStreamId, {
|
|
98
100
|
...options.checksumOptions,
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
checksumCacheTtlMs: options.checksumCacheTtlMs,
|
|
102
|
+
storageConfig: options?.storageConfig
|
|
101
103
|
});
|
|
102
104
|
}
|
|
103
105
|
createMongoCompactor(options) {
|
|
104
106
|
return new MongoCompactorV1(this, this.db, options);
|
|
105
107
|
}
|
|
106
108
|
createMongoParameterCompactor(checkpoint, options) {
|
|
107
|
-
return new MongoParameterCompactorV1(this.db, this.
|
|
109
|
+
return new MongoParameterCompactorV1(this.db, this.replicationStreamId, checkpoint, options);
|
|
108
110
|
}
|
|
109
111
|
get versionContext() {
|
|
110
112
|
return {
|
|
111
113
|
db: this.db,
|
|
112
|
-
|
|
113
|
-
|
|
114
|
+
replicationStreamId: this.replicationStreamId,
|
|
115
|
+
readPreference: this.readPreference
|
|
114
116
|
};
|
|
115
117
|
}
|
|
116
118
|
getParameterSetsImpl(checkpoint, lookups, limit) {
|
|
@@ -120,29 +122,37 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
120
122
|
return getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options);
|
|
121
123
|
}
|
|
122
124
|
async clearBucketData(signal) {
|
|
123
|
-
await this.
|
|
124
|
-
_id: idPrefixFilter({ g: this.
|
|
125
|
-
},
|
|
125
|
+
await clearCollectionInIdRanges(this.logger, 'bucket data', this.db.bucket_data, {
|
|
126
|
+
_id: idPrefixFilter({ g: this.replicationStreamId }, ['b', 'o'])
|
|
127
|
+
}, signal, this.clearBatchThrottleRate);
|
|
126
128
|
}
|
|
127
129
|
async clearParameterIndexes(signal) {
|
|
128
|
-
await this.
|
|
129
|
-
'key.g': this.
|
|
130
|
-
},
|
|
130
|
+
await clearCollectionInIdBatches(this.logger, 'parameter index', this.db.parameterIndexV1, {
|
|
131
|
+
'key.g': this.replicationStreamId
|
|
132
|
+
}, signal, this.clearBatchThrottleRate);
|
|
131
133
|
}
|
|
132
134
|
async clearSourceRecords(signal) {
|
|
133
|
-
await this.
|
|
134
|
-
_id: idPrefixFilter({ g: this.
|
|
135
|
-
},
|
|
135
|
+
await clearCollectionInIdRanges(this.logger, 'source records', this.db.sourceRecordsV1, {
|
|
136
|
+
_id: idPrefixFilter({ g: this.replicationStreamId }, ['t', 'k'])
|
|
137
|
+
}, signal, this.clearBatchThrottleRate);
|
|
136
138
|
}
|
|
137
139
|
async clearBucketState(signal) {
|
|
138
|
-
await this.
|
|
139
|
-
_id: idPrefixFilter({ g: this.
|
|
140
|
-
},
|
|
140
|
+
await clearCollectionInIdRanges(this.logger, 'bucket state', this.db.bucketStateV1, {
|
|
141
|
+
_id: idPrefixFilter({ g: this.replicationStreamId }, ['b'])
|
|
142
|
+
}, signal, this.clearBatchThrottleRate);
|
|
141
143
|
}
|
|
142
144
|
async clearSourceTables(signal) {
|
|
143
|
-
await
|
|
144
|
-
group_id: this.
|
|
145
|
-
}, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }),
|
|
145
|
+
await retryOnMongoMaxTimeMSExpired(() => this.db.sourceTablesV1(this.replicationStreamId).deleteMany({
|
|
146
|
+
group_id: this.replicationStreamId
|
|
147
|
+
}, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }), {
|
|
148
|
+
signal,
|
|
149
|
+
abortMessage: 'Aborted clearing data',
|
|
150
|
+
// This is a fairly long delay - only expected to hit this when the storage database is under high load.
|
|
151
|
+
retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
|
|
152
|
+
onRetry: () => {
|
|
153
|
+
this.logger.info(`Clearing batch of source tables timed out after ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, retrying...`);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
146
156
|
}
|
|
147
157
|
getDataBucketChangesImpl(options) {
|
|
148
158
|
return getDataBucketChangesV1(this.versionContext, options);
|
|
@@ -161,7 +171,7 @@ export async function getParameterSetsV1(ctx, checkpoint, lookups, limit) {
|
|
|
161
171
|
.aggregate([
|
|
162
172
|
{
|
|
163
173
|
$match: {
|
|
164
|
-
'key.g': ctx.
|
|
174
|
+
'key.g': ctx.replicationStreamId,
|
|
165
175
|
lookup: { $in: lookupFilter },
|
|
166
176
|
_id: { $lte: checkpoint.checkpoint }
|
|
167
177
|
}
|
|
@@ -217,22 +227,32 @@ export async function* getBucketDataBatchV1(ctx, checkpoint, dataBuckets, option
|
|
|
217
227
|
if (dataBuckets.length == 0) {
|
|
218
228
|
return;
|
|
219
229
|
}
|
|
230
|
+
const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
|
|
231
|
+
const readConcern = ctx.readPreference == null ? undefined : 'majority';
|
|
232
|
+
const session = readPreference == null || checkpoint.snapshotTime == null
|
|
233
|
+
? undefined
|
|
234
|
+
: ctx.db.client.startSession({ causalConsistency: true });
|
|
235
|
+
await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
|
|
236
|
+
if (session != null) {
|
|
237
|
+
session.advanceOperationTime(checkpoint.snapshotTime);
|
|
238
|
+
session.advanceClusterTime(checkpoint.clusterTime);
|
|
239
|
+
}
|
|
220
240
|
let filters = [];
|
|
221
241
|
const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
|
|
222
|
-
if (checkpoint == null) {
|
|
242
|
+
if (checkpoint.checkpoint == null) {
|
|
223
243
|
throw new Error('checkpoint is null');
|
|
224
244
|
}
|
|
225
|
-
const end = checkpoint;
|
|
245
|
+
const end = checkpoint.checkpoint;
|
|
226
246
|
for (let { bucket: name, start } of dataBuckets) {
|
|
227
247
|
filters.push({
|
|
228
248
|
_id: {
|
|
229
249
|
$gt: {
|
|
230
|
-
g: ctx.
|
|
250
|
+
g: ctx.replicationStreamId,
|
|
231
251
|
b: name,
|
|
232
252
|
o: start
|
|
233
253
|
},
|
|
234
254
|
$lte: {
|
|
235
|
-
g: ctx.
|
|
255
|
+
g: ctx.replicationStreamId,
|
|
236
256
|
b: name,
|
|
237
257
|
o: end
|
|
238
258
|
}
|
|
@@ -241,10 +261,14 @@ export async function* getBucketDataBatchV1(ctx, checkpoint, dataBuckets, option
|
|
|
241
261
|
}
|
|
242
262
|
const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
|
|
243
263
|
const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
|
|
264
|
+
// raw: true returns Buffers, but the driver typing doesn't reflect that
|
|
265
|
+
// without an explicit cast to FindCursor<Buffer>.
|
|
244
266
|
const cursor = ctx.db.bucket_data.find({
|
|
245
267
|
$or: filters
|
|
246
268
|
}, {
|
|
247
|
-
session
|
|
269
|
+
session,
|
|
270
|
+
readPreference,
|
|
271
|
+
readConcern,
|
|
248
272
|
sort: { _id: 1 },
|
|
249
273
|
limit: batchLimit,
|
|
250
274
|
batchSize: batchLimit + 1,
|
|
@@ -310,7 +334,7 @@ export async function getDataBucketChangesV1(ctx, options) {
|
|
|
310
334
|
const limit = 1000;
|
|
311
335
|
const bucketStateUpdates = await ctx.db.bucketStateV1
|
|
312
336
|
.find({
|
|
313
|
-
'_id.g': ctx.
|
|
337
|
+
'_id.g': ctx.replicationStreamId,
|
|
314
338
|
last_op: { $gt: options.lastCheckpoint.checkpoint }
|
|
315
339
|
}, {
|
|
316
340
|
projection: {
|
|
@@ -333,7 +357,7 @@ export async function getParameterBucketChangesV1(ctx, options) {
|
|
|
333
357
|
const parameterUpdates = await ctx.db.parameterIndexV1
|
|
334
358
|
.find({
|
|
335
359
|
_id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
|
|
336
|
-
'key.g': ctx.
|
|
360
|
+
'key.g': ctx.replicationStreamId
|
|
337
361
|
}, {
|
|
338
362
|
projection: {
|
|
339
363
|
lookup: 1
|