@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
-
import { storage } from '@powersync/service-core';
|
|
3
|
+
import { addChecksums, storage, utils } from '@powersync/service-core';
|
|
4
4
|
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
5
|
-
import {
|
|
5
|
+
import { BucketDataDoc } from '../common/BucketDataDoc.js';
|
|
6
6
|
import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
|
|
7
|
-
import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
|
|
8
|
-
import {
|
|
7
|
+
import { CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
|
|
8
|
+
import { cacheKey } from '../OperationBatch.js';
|
|
9
|
+
import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js';
|
|
9
10
|
import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
|
|
10
11
|
import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
|
|
11
12
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
12
13
|
|
|
14
|
+
type CompactClearProperties = 'op' | 'checksum' | 'target_op';
|
|
15
|
+
|
|
13
16
|
export class MongoCompactorV1 extends MongoCompactor {
|
|
14
17
|
// Override types to the more specific ones
|
|
15
18
|
declare protected readonly db: VersionedPowerSyncMongoV1;
|
|
16
19
|
declare protected readonly storage: MongoSyncBucketStorageV1;
|
|
17
20
|
|
|
21
|
+
private updates: mongo.AnyBulkWriteOperation<BucketDataDocumentV1>[] = [];
|
|
22
|
+
|
|
18
23
|
public async *dirtyBucketBatches(options: {
|
|
19
24
|
minBucketChanges: number;
|
|
20
25
|
minChangeRatio: number;
|
|
@@ -64,7 +69,8 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
64
69
|
buckets.map(({ bucket }) => ({
|
|
65
70
|
bucket,
|
|
66
71
|
end: this.maxOpId
|
|
67
|
-
}))
|
|
72
|
+
})),
|
|
73
|
+
{ readOptions: { readConcern: 'snapshot' } }
|
|
68
74
|
);
|
|
69
75
|
}
|
|
70
76
|
|
|
@@ -80,14 +86,293 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
80
86
|
};
|
|
81
87
|
}
|
|
82
88
|
|
|
83
|
-
|
|
84
|
-
bucket: string,
|
|
85
|
-
_definitionId: BucketDefinitionId | null
|
|
86
|
-
): Promise<SingleBucketStore | null> {
|
|
89
|
+
private getBucketDataContext(bucket: string): SingleBucketStoreV1 {
|
|
87
90
|
return new SingleBucketStoreV1(this.db, {
|
|
88
91
|
replicationStreamId: this.group_id,
|
|
89
92
|
definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
|
|
90
93
|
bucket
|
|
91
94
|
});
|
|
92
95
|
}
|
|
96
|
+
|
|
97
|
+
protected async compactSingleBucket(bucket: string) {
|
|
98
|
+
// Do not carry queued writes from a failed attempt into the rescan
|
|
99
|
+
this.updates = [];
|
|
100
|
+
|
|
101
|
+
const idLimitBytes = this.idLimitBytes;
|
|
102
|
+
const bucketContext = this.getBucketDataContext(bucket);
|
|
103
|
+
const currentState: CurrentBucketState = {
|
|
104
|
+
bucket,
|
|
105
|
+
definitionId: bucketContext.key.definitionId,
|
|
106
|
+
seen: new Map(),
|
|
107
|
+
trackingSize: 0,
|
|
108
|
+
lastNotPut: null,
|
|
109
|
+
opsSincePut: 0,
|
|
110
|
+
checksum: 0,
|
|
111
|
+
opCount: 0,
|
|
112
|
+
opBytes: 0
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
// Constant lower bound.
|
|
116
|
+
const lowerBound = bucketContext.minId;
|
|
117
|
+
// Upper bound is adjusted for each batch.
|
|
118
|
+
let upperBound = bucketContext.maxId;
|
|
119
|
+
|
|
120
|
+
// Paginate through bucket data in batches to avoid cursor timeouts.
|
|
121
|
+
while (true) {
|
|
122
|
+
this.signal?.throwIfAborted();
|
|
123
|
+
|
|
124
|
+
// Query one batch at a time, to avoid cursor timeouts.
|
|
125
|
+
const pipeline = [
|
|
126
|
+
{
|
|
127
|
+
$match: {
|
|
128
|
+
_id: {
|
|
129
|
+
$gte: lowerBound,
|
|
130
|
+
$lt: upperBound
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
{ $sort: { _id: -1 } },
|
|
135
|
+
{ $limit: this.moveBatchQueryLimit },
|
|
136
|
+
{
|
|
137
|
+
$project: {
|
|
138
|
+
_id: 1,
|
|
139
|
+
op: 1,
|
|
140
|
+
table: 1,
|
|
141
|
+
row_id: 1,
|
|
142
|
+
source_table: 1,
|
|
143
|
+
source_key: 1,
|
|
144
|
+
checksum: 1,
|
|
145
|
+
size: { $bsonSize: '$$ROOT' }
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
];
|
|
149
|
+
|
|
150
|
+
const cursor = bucketContext.collection.aggregate<BucketDataDocumentV1 & { size: number | bigint }>(pipeline, {
|
|
151
|
+
// batchSize is 1 more than limit to auto-close the cursor.
|
|
152
|
+
// See https://github.com/mongodb/node-mongodb-native/pull/4580
|
|
153
|
+
batchSize: this.moveBatchQueryLimit + 1
|
|
154
|
+
});
|
|
155
|
+
// We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
|
|
156
|
+
// Instead, we load up to the limit.
|
|
157
|
+
const rawBatch = await cursor.toArray();
|
|
158
|
+
const batch = rawBatch.map((document) => {
|
|
159
|
+
const { size, ...rest } = document;
|
|
160
|
+
return {
|
|
161
|
+
doc: bucketContext.fromPersistedDocument(rest),
|
|
162
|
+
size
|
|
163
|
+
};
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
if (batch.length == 0) {
|
|
167
|
+
// We've reached the end.
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Reuse the exact collection _id value from Mongo for the next bound.
|
|
172
|
+
upperBound = rawBatch[rawBatch.length - 1]._id;
|
|
173
|
+
|
|
174
|
+
for (const { doc, size } of batch) {
|
|
175
|
+
if (doc.o > this.maxOpId) {
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
|
|
180
|
+
currentState.opCount += 1;
|
|
181
|
+
|
|
182
|
+
let isPersistentPut = doc.op == 'PUT';
|
|
183
|
+
|
|
184
|
+
currentState.opBytes += Number(size);
|
|
185
|
+
if (doc.op == 'REMOVE' || doc.op == 'PUT') {
|
|
186
|
+
const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`;
|
|
187
|
+
const targetOp = currentState.seen.get(key);
|
|
188
|
+
if (targetOp) {
|
|
189
|
+
// Will convert to MOVE, so don't count as PUT.
|
|
190
|
+
isPersistentPut = false;
|
|
191
|
+
|
|
192
|
+
this.updates.push({
|
|
193
|
+
updateOne: {
|
|
194
|
+
filter: { _id: bucketContext.docId(doc.o) },
|
|
195
|
+
update: {
|
|
196
|
+
$set: {
|
|
197
|
+
op: 'MOVE',
|
|
198
|
+
target_op: targetOp
|
|
199
|
+
},
|
|
200
|
+
$unset: {
|
|
201
|
+
source_table: 1,
|
|
202
|
+
source_key: 1,
|
|
203
|
+
table: 1,
|
|
204
|
+
row_id: 1,
|
|
205
|
+
data: 1
|
|
206
|
+
}
|
|
207
|
+
} satisfies mongo.UpdateFilter<BucketDataDocumentV1>
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
// TODO: better estimate for this.
|
|
212
|
+
currentState.opBytes += 200 - Number(size);
|
|
213
|
+
} else if (currentState.trackingSize < idLimitBytes) {
|
|
214
|
+
// flatstr reduces the memory usage by flattening the string.
|
|
215
|
+
currentState.seen.set(utils.flatstr(key), doc.o);
|
|
216
|
+
// length + 16 for the string
|
|
217
|
+
// 24 for the bigint
|
|
218
|
+
// 50 for map overhead
|
|
219
|
+
// 50 for additional overhead
|
|
220
|
+
currentState.trackingSize += key.length + 140;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (isPersistentPut) {
|
|
225
|
+
currentState.lastNotPut = null;
|
|
226
|
+
currentState.opsSincePut = 0;
|
|
227
|
+
} else if (doc.op != 'CLEAR') {
|
|
228
|
+
if (currentState.lastNotPut == null) {
|
|
229
|
+
currentState.lastNotPut = doc.o;
|
|
230
|
+
}
|
|
231
|
+
currentState.opsSincePut += 1;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
|
|
235
|
+
await this.flush(bucketContext);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Free memory before clearing the bucket.
|
|
243
|
+
currentState.seen.clear();
|
|
244
|
+
if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
|
|
245
|
+
this.logger.info(
|
|
246
|
+
`Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
|
|
247
|
+
);
|
|
248
|
+
// Need flush() before clear().
|
|
249
|
+
await this.flush(bucketContext);
|
|
250
|
+
await this.clearBucket(currentState, bucketContext);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Do this after clearBucket so we have accurate counts.
|
|
254
|
+
this.updateBucketChecksums(currentState, this.maxOpId);
|
|
255
|
+
// Need another flush after updateBucketChecksums().
|
|
256
|
+
await this.flush(bucketContext);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private async flush(bucketContext: SingleBucketStoreV1) {
|
|
260
|
+
if (this.updates.length > 0) {
|
|
261
|
+
this.logger.info(`Compacting ${this.updates.length} ops`);
|
|
262
|
+
await bucketContext.collection.bulkWrite(this.updates, {
|
|
263
|
+
// Order is not important. Since checksums are not affected, these operations can happen in any order,
|
|
264
|
+
// and it's fine if the operations are partially applied. Each individual operation is atomic.
|
|
265
|
+
ordered: false
|
|
266
|
+
});
|
|
267
|
+
this.updates = [];
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
await this.flushBucketStateUpdates();
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Perform a CLEAR compact for a bucket.
|
|
275
|
+
*
|
|
276
|
+
* @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
|
|
277
|
+
*/
|
|
278
|
+
private async clearBucket(currentState: CurrentBucketState, bucketContext: SingleBucketStoreV1) {
|
|
279
|
+
const clearOp = currentState.lastNotPut!;
|
|
280
|
+
|
|
281
|
+
const opFilter = {
|
|
282
|
+
_id: {
|
|
283
|
+
$gte: bucketContext.minId,
|
|
284
|
+
$lte: bucketContext.docId(clearOp)
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
|
|
288
|
+
const session = this.db.client.startSession();
|
|
289
|
+
try {
|
|
290
|
+
let done = false;
|
|
291
|
+
while (!done) {
|
|
292
|
+
this.signal?.throwIfAborted();
|
|
293
|
+
let opCountDiff = 0;
|
|
294
|
+
// Do the CLEAR operation in batches, with each batch a separate transaction.
|
|
295
|
+
// The state after each batch is fully consistent.
|
|
296
|
+
// We need a transaction per batch to make sure checksums stay consistent.
|
|
297
|
+
await session.withTransaction(
|
|
298
|
+
async () => {
|
|
299
|
+
const query = bucketContext.collection.find<Pick<BucketDataDocumentV1, '_id' | CompactClearProperties>>(
|
|
300
|
+
opFilter,
|
|
301
|
+
{
|
|
302
|
+
session,
|
|
303
|
+
sort: { _id: 1 },
|
|
304
|
+
projection: {
|
|
305
|
+
_id: 1,
|
|
306
|
+
op: 1,
|
|
307
|
+
checksum: 1,
|
|
308
|
+
target_op: 1
|
|
309
|
+
},
|
|
310
|
+
limit: this.clearBatchLimit
|
|
311
|
+
}
|
|
312
|
+
);
|
|
313
|
+
let checksum = 0;
|
|
314
|
+
let lastOp: Pick<BucketDataDoc, 'o' | CompactClearProperties> | null = null;
|
|
315
|
+
let targetOp: bigint | null = null;
|
|
316
|
+
let gotAnOp = false;
|
|
317
|
+
let numberOfOpsToClear = 0;
|
|
318
|
+
for await (const rawOp of query.stream()) {
|
|
319
|
+
const op = bucketContext.fromPartialPersistedDocument(rawOp);
|
|
320
|
+
|
|
321
|
+
if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
|
|
322
|
+
checksum = utils.addChecksums(checksum, Number(op.checksum));
|
|
323
|
+
lastOp = op;
|
|
324
|
+
numberOfOpsToClear += 1;
|
|
325
|
+
if (op.op != 'CLEAR') {
|
|
326
|
+
gotAnOp = true;
|
|
327
|
+
}
|
|
328
|
+
if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
|
|
329
|
+
targetOp = op.target_op;
|
|
330
|
+
}
|
|
331
|
+
} else {
|
|
332
|
+
throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
if (!gotAnOp) {
|
|
336
|
+
done = true;
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
|
|
341
|
+
await bucketContext.collection.deleteMany(
|
|
342
|
+
{
|
|
343
|
+
_id: {
|
|
344
|
+
$gte: bucketContext.minId,
|
|
345
|
+
$lte: bucketContext.docId(lastOp!.o)
|
|
346
|
+
}
|
|
347
|
+
},
|
|
348
|
+
{ session }
|
|
349
|
+
);
|
|
350
|
+
|
|
351
|
+
const op = bucketContext.toPersistedDocument({
|
|
352
|
+
o: lastOp!.o,
|
|
353
|
+
op: 'CLEAR',
|
|
354
|
+
checksum: BigInt(checksum),
|
|
355
|
+
data: null,
|
|
356
|
+
target_op: targetOp
|
|
357
|
+
});
|
|
358
|
+
await bucketContext.collection.insertOne(op, { session });
|
|
359
|
+
|
|
360
|
+
opCountDiff = -numberOfOpsToClear + 1;
|
|
361
|
+
},
|
|
362
|
+
{
|
|
363
|
+
writeConcern: { w: 'majority' },
|
|
364
|
+
readConcern: { level: 'snapshot' }
|
|
365
|
+
}
|
|
366
|
+
);
|
|
367
|
+
// Update outside the transaction, since the transaction can be retried multiple times.
|
|
368
|
+
currentState.opCount += opCountDiff;
|
|
369
|
+
}
|
|
370
|
+
} finally {
|
|
371
|
+
await session.endSession();
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
private formatBucketDataKey(doc: Pick<BucketDataDoc, 'bucketKey' | 'o'>) {
|
|
376
|
+
return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
|
|
377
|
+
}
|
|
93
378
|
}
|