@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,9 +1,55 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
1
|
+
import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { addChecksums, isPartialChecksum, SyncRuleState, utils } from '@powersync/service-core';
|
|
3
4
|
import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
|
|
4
5
|
import { MongoCompactor } from '../MongoCompactor.js';
|
|
6
|
+
import { cacheKey } from '../OperationBatch.js';
|
|
5
7
|
import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
|
|
8
|
+
const DEFAULT_MIN_BUCKET_CHANGES = 10;
|
|
9
|
+
const DEFAULT_MIN_CHANGE_RATIO = 0.1;
|
|
10
|
+
const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
|
|
6
11
|
export class MongoCompactorV1 extends MongoCompactor {
|
|
12
|
+
options;
|
|
13
|
+
updates = [];
|
|
14
|
+
bucketStateUpdates = [];
|
|
15
|
+
minBucketChanges;
|
|
16
|
+
minChangeRatio;
|
|
17
|
+
maxOpId;
|
|
18
|
+
constructor(bucketStorage, db, options) {
|
|
19
|
+
super(bucketStorage, db, options);
|
|
20
|
+
this.options = options;
|
|
21
|
+
this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
|
|
22
|
+
this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
|
|
23
|
+
this.maxOpId = options.maxOpId ?? 0n;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
27
|
+
*
|
|
28
|
+
* See /docs/storage/compacting-operations.md for details.
|
|
29
|
+
*/
|
|
30
|
+
async compact() {
|
|
31
|
+
if (this.options?.incrementalOnly) {
|
|
32
|
+
// Not supported for V1
|
|
33
|
+
this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2');
|
|
34
|
+
return 0;
|
|
35
|
+
}
|
|
36
|
+
else if (this.storage.replicationStream.state != SyncRuleState.ACTIVE) {
|
|
37
|
+
this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`);
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
await this.deleteOldCheckpointRequests();
|
|
41
|
+
if (this.buckets) {
|
|
42
|
+
for (const bucket of this.buckets) {
|
|
43
|
+
// We can make this more efficient later on by iterating through the buckets in a single query.
|
|
44
|
+
// That makes batching more tricky, so we leave for later.
|
|
45
|
+
await this.compactSingleBucketRetried(bucket);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
await this.compactDirtyBuckets();
|
|
50
|
+
}
|
|
51
|
+
return this.compactedBucketCount;
|
|
52
|
+
}
|
|
7
53
|
async *dirtyBucketBatches(options) {
|
|
8
54
|
if (options.minBucketChanges <= 0) {
|
|
9
55
|
throw new ReplicationAssertionError('minBucketChanges must be >= 1');
|
|
@@ -11,7 +57,7 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
11
57
|
// Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes.
|
|
12
58
|
// That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is
|
|
13
59
|
// still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting.
|
|
14
|
-
yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options
|
|
60
|
+
yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options);
|
|
15
61
|
}
|
|
16
62
|
async dirtyBucketBatchForChecksums(options) {
|
|
17
63
|
if (options.minBucketChanges <= 0) {
|
|
@@ -19,10 +65,186 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
19
65
|
}
|
|
20
66
|
// Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
|
|
21
67
|
// estimate_since_compact as it progresses.
|
|
22
|
-
return this.dirtyBucketBatchForChecksumsForCollection(
|
|
68
|
+
return this.dirtyBucketBatchForChecksumsForCollection({
|
|
23
69
|
'_id.g': this.group_id,
|
|
24
70
|
'estimate_since_compact.count': { $gte: options.minBucketChanges }
|
|
25
|
-
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
async *dirtyBucketBatchesForCollection(collection, lastId, maxId, options) {
|
|
74
|
+
// Paginate through the bucket state collection using cursor-based scanning.
|
|
75
|
+
while (true) {
|
|
76
|
+
// To avoid timeouts from too many buckets not meeting the minimum-change
|
|
77
|
+
// criteria, scan a fixed batch and only return matching buckets.
|
|
78
|
+
const [result] = await collection
|
|
79
|
+
.aggregate([
|
|
80
|
+
{ $match: { _id: { $gt: lastId, $lt: maxId } } },
|
|
81
|
+
{ $sort: { _id: 1 } },
|
|
82
|
+
// Scan a fixed number of docs each query so sparse matches don't block progress.
|
|
83
|
+
{ $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE },
|
|
84
|
+
{
|
|
85
|
+
$facet: {
|
|
86
|
+
buckets: [
|
|
87
|
+
{ $match: { 'estimate_since_compact.count': { $gte: options.minBucketChanges } } },
|
|
88
|
+
{ $project: { _id: 1, estimate_since_compact: 1, compacted_state: 1 } }
|
|
89
|
+
],
|
|
90
|
+
cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
], { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS })
|
|
94
|
+
.toArray();
|
|
95
|
+
const cursor = result?.cursor?.[0];
|
|
96
|
+
if (cursor == null) {
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
lastId = cursor._id;
|
|
100
|
+
const dirtyBuckets = (result?.buckets ?? []).map((bucketState) => {
|
|
101
|
+
// BigInt precision is not needed here since this is only an estimate.
|
|
102
|
+
const updatedCount = bucketState.estimate_since_compact?.count ?? 0;
|
|
103
|
+
const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount;
|
|
104
|
+
const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0);
|
|
105
|
+
const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes;
|
|
106
|
+
return {
|
|
107
|
+
bucket: bucketState._id.b,
|
|
108
|
+
definitionId: null,
|
|
109
|
+
estimatedCount: totalCount,
|
|
110
|
+
dirtyRatio: Math.max(totalCount > 0 ? updatedCount / totalCount : 0, totalBytes > 0 ? updatedBytes / totalBytes : 0)
|
|
111
|
+
};
|
|
112
|
+
});
|
|
113
|
+
yield dirtyBuckets.filter((bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
async dirtyBucketBatchForChecksumsForCollection(filter) {
|
|
117
|
+
const dirtyBuckets = await this.db.bucketStateV1
|
|
118
|
+
.find(filter, {
|
|
119
|
+
projection: { _id: 1, estimate_since_compact: 1, compacted_state: 1 },
|
|
120
|
+
sort: { 'estimate_since_compact.count': -1 },
|
|
121
|
+
limit: 200,
|
|
122
|
+
maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
|
|
123
|
+
})
|
|
124
|
+
.toArray();
|
|
125
|
+
return dirtyBuckets.map((bucket) => ({
|
|
126
|
+
bucket: bucket._id.b,
|
|
127
|
+
definitionId: null,
|
|
128
|
+
estimatedCount: Number(bucket.estimate_since_compact.count) + Number(bucket.compacted_state?.count ?? 0)
|
|
129
|
+
}));
|
|
130
|
+
}
|
|
131
|
+
async compactSingleBucketRetried(bucket, _definitionId = null) {
|
|
132
|
+
await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket));
|
|
133
|
+
this.compactedBucketCount++;
|
|
134
|
+
}
|
|
135
|
+
async compactDirtyBuckets() {
|
|
136
|
+
for await (const buckets of this.dirtyBucketBatches({
|
|
137
|
+
minBucketChanges: this.minBucketChanges,
|
|
138
|
+
minChangeRatio: this.minChangeRatio
|
|
139
|
+
})) {
|
|
140
|
+
this.signal?.throwIfAborted();
|
|
141
|
+
for (const { bucket, definitionId } of buckets) {
|
|
142
|
+
await this.compactSingleBucketRetried(bucket, definitionId);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Subset of compact, only populating checksums where relevant.
|
|
148
|
+
*/
|
|
149
|
+
async populateChecksums(options) {
|
|
150
|
+
let count = 0;
|
|
151
|
+
// Paginate through dirty buckets in batches until no more buckets meet the criteria.
|
|
152
|
+
while (true) {
|
|
153
|
+
this.signal?.throwIfAborted();
|
|
154
|
+
const buckets = await this.dirtyBucketBatchForChecksums(options);
|
|
155
|
+
if (buckets.length == 0) {
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
this.signal?.throwIfAborted();
|
|
159
|
+
const start = Date.now();
|
|
160
|
+
// Filter batch by estimated bucket size, to reduce possibility of timeouts.
|
|
161
|
+
const checkBuckets = [];
|
|
162
|
+
let totalCountEstimate = 0;
|
|
163
|
+
for (const bucket of buckets) {
|
|
164
|
+
checkBuckets.push(bucket);
|
|
165
|
+
totalCountEstimate += bucket.estimatedCount;
|
|
166
|
+
if (totalCountEstimate > 50_000) {
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
this.logger.info(`Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`);
|
|
171
|
+
await this.updateChecksumsBatch(checkBuckets);
|
|
172
|
+
this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
|
|
173
|
+
count += checkBuckets.length;
|
|
174
|
+
}
|
|
175
|
+
return { buckets: count };
|
|
176
|
+
}
|
|
177
|
+
collectBucketStateUpdate(state, compactedOpId) {
|
|
178
|
+
if (state.opCount < 0) {
|
|
179
|
+
throw new ServiceAssertionError(`Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`);
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
updateOne: {
|
|
183
|
+
filter: this.bucketStateFilter(state.bucket),
|
|
184
|
+
update: {
|
|
185
|
+
$set: {
|
|
186
|
+
compacted_state: {
|
|
187
|
+
op_id: compactedOpId,
|
|
188
|
+
count: state.opCount,
|
|
189
|
+
checksum: BigInt(state.checksum),
|
|
190
|
+
bytes: state.opBytes
|
|
191
|
+
},
|
|
192
|
+
estimate_since_compact: {
|
|
193
|
+
// There could have been a whole bunch of new operations added to the bucket while compacting,
|
|
194
|
+
// which we don't currently cater for. We could potentially query for that, but that adds overhead.
|
|
195
|
+
count: 0,
|
|
196
|
+
bytes: 0
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
// We generally expect this to have been created before.
|
|
201
|
+
// We don't create new ones here, to avoid issues with the unique index on bucket_updates.
|
|
202
|
+
upsert: false
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
updateBucketChecksums(state, compactedOpId) {
|
|
207
|
+
this.bucketStateUpdates.push(this.collectBucketStateUpdate(state, compactedOpId));
|
|
208
|
+
}
|
|
209
|
+
async flushBucketStateUpdates() {
|
|
210
|
+
if (this.bucketStateUpdates.length > 0) {
|
|
211
|
+
this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
|
|
212
|
+
await this.writeBucketStateUpdates();
|
|
213
|
+
this.bucketStateUpdates = [];
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async updateChecksumsBatch(buckets) {
|
|
217
|
+
const checksums = await this.computeChecksumsForBuckets(buckets);
|
|
218
|
+
const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
|
|
219
|
+
for (const bucketChecksum of checksums.values()) {
|
|
220
|
+
if (isPartialChecksum(bucketChecksum)) {
|
|
221
|
+
// Should never happen since we don't specify `start`.
|
|
222
|
+
throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`);
|
|
223
|
+
}
|
|
224
|
+
this.bucketStateUpdates.push({
|
|
225
|
+
updateOne: {
|
|
226
|
+
filter: this.bucketStateFilter(bucketChecksum.bucket),
|
|
227
|
+
update: {
|
|
228
|
+
$set: {
|
|
229
|
+
compacted_state: {
|
|
230
|
+
op_id: this.maxOpId,
|
|
231
|
+
count: bucketChecksum.count,
|
|
232
|
+
checksum: BigInt(bucketChecksum.checksum),
|
|
233
|
+
bytes: null
|
|
234
|
+
},
|
|
235
|
+
estimate_since_compact: {
|
|
236
|
+
count: 0,
|
|
237
|
+
bytes: 0
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
},
|
|
241
|
+
// We don't create new ones here - it gets tricky to get the last_op right with the unique index on
|
|
242
|
+
// bucket_updates.
|
|
243
|
+
upsert: false
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
await this.flushBucketStateUpdates();
|
|
26
248
|
}
|
|
27
249
|
async writeBucketStateUpdates() {
|
|
28
250
|
await this.db.bucketStateV1.bulkWrite(this.bucketStateUpdates, { ordered: false });
|
|
@@ -31,9 +253,9 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
31
253
|
return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
|
|
32
254
|
bucket,
|
|
33
255
|
end: this.maxOpId
|
|
34
|
-
})));
|
|
256
|
+
})), { readOptions: { readConcern: 'snapshot' } });
|
|
35
257
|
}
|
|
36
|
-
bucketStateFilter(bucket
|
|
258
|
+
bucketStateFilter(bucket) {
|
|
37
259
|
return {
|
|
38
260
|
_id: {
|
|
39
261
|
g: this.group_id,
|
|
@@ -41,12 +263,260 @@ export class MongoCompactorV1 extends MongoCompactor {
|
|
|
41
263
|
}
|
|
42
264
|
};
|
|
43
265
|
}
|
|
44
|
-
|
|
266
|
+
getBucketDataContext(bucket) {
|
|
45
267
|
return new SingleBucketStoreV1(this.db, {
|
|
46
268
|
replicationStreamId: this.group_id,
|
|
47
269
|
definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
|
|
48
270
|
bucket
|
|
49
271
|
});
|
|
50
272
|
}
|
|
273
|
+
async compactSingleBucket(bucket) {
|
|
274
|
+
// Do not carry queued writes from a failed attempt into the rescan.
|
|
275
|
+
this.updates = [];
|
|
276
|
+
this.bucketStateUpdates = [];
|
|
277
|
+
const idLimitBytes = this.idLimitBytes;
|
|
278
|
+
const bucketContext = this.getBucketDataContext(bucket);
|
|
279
|
+
const currentState = {
|
|
280
|
+
bucket,
|
|
281
|
+
definitionId: bucketContext.key.definitionId,
|
|
282
|
+
seen: new Map(),
|
|
283
|
+
trackingSize: 0,
|
|
284
|
+
lastNotPut: null,
|
|
285
|
+
opsSincePut: 0,
|
|
286
|
+
checksum: 0,
|
|
287
|
+
opCount: 0,
|
|
288
|
+
opBytes: 0
|
|
289
|
+
};
|
|
290
|
+
// Constant lower bound.
|
|
291
|
+
const lowerBound = bucketContext.minId;
|
|
292
|
+
// Upper bound is adjusted for each batch.
|
|
293
|
+
let upperBound = bucketContext.maxId;
|
|
294
|
+
// Paginate through bucket data in batches to avoid cursor timeouts.
|
|
295
|
+
while (true) {
|
|
296
|
+
this.signal?.throwIfAborted();
|
|
297
|
+
// Query one batch at a time, to avoid cursor timeouts.
|
|
298
|
+
const pipeline = [
|
|
299
|
+
{
|
|
300
|
+
$match: {
|
|
301
|
+
_id: {
|
|
302
|
+
$gte: lowerBound,
|
|
303
|
+
$lt: upperBound
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
},
|
|
307
|
+
{ $sort: { _id: -1 } },
|
|
308
|
+
{ $limit: this.moveBatchQueryLimit },
|
|
309
|
+
{
|
|
310
|
+
$project: {
|
|
311
|
+
_id: 1,
|
|
312
|
+
op: 1,
|
|
313
|
+
table: 1,
|
|
314
|
+
row_id: 1,
|
|
315
|
+
source_table: 1,
|
|
316
|
+
source_key: 1,
|
|
317
|
+
checksum: 1,
|
|
318
|
+
size: { $bsonSize: '$$ROOT' }
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
];
|
|
322
|
+
const cursor = bucketContext.collection.aggregate(pipeline, {
|
|
323
|
+
// batchSize is 1 more than limit to auto-close the cursor.
|
|
324
|
+
// See https://github.com/mongodb/node-mongodb-native/pull/4580
|
|
325
|
+
batchSize: this.moveBatchQueryLimit + 1
|
|
326
|
+
});
|
|
327
|
+
// We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
|
|
328
|
+
// Instead, we load up to the limit.
|
|
329
|
+
const rawBatch = await cursor.toArray();
|
|
330
|
+
const batch = rawBatch.map((document) => {
|
|
331
|
+
const { size, ...rest } = document;
|
|
332
|
+
return {
|
|
333
|
+
doc: bucketContext.fromPersistedDocument(rest),
|
|
334
|
+
size
|
|
335
|
+
};
|
|
336
|
+
});
|
|
337
|
+
if (batch.length == 0) {
|
|
338
|
+
// We've reached the end.
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
// Reuse the exact collection _id value from Mongo for the next bound.
|
|
342
|
+
upperBound = rawBatch[rawBatch.length - 1]._id;
|
|
343
|
+
for (const { doc, size } of batch) {
|
|
344
|
+
if (doc.o > this.maxOpId) {
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
|
|
348
|
+
currentState.opCount += 1;
|
|
349
|
+
let isPersistentPut = doc.op == 'PUT';
|
|
350
|
+
currentState.opBytes += Number(size);
|
|
351
|
+
if (doc.op == 'REMOVE' || doc.op == 'PUT') {
|
|
352
|
+
const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table, doc.source_key)}`;
|
|
353
|
+
const targetOp = currentState.seen.get(key);
|
|
354
|
+
if (targetOp) {
|
|
355
|
+
// Will convert to MOVE, so don't count as PUT.
|
|
356
|
+
isPersistentPut = false;
|
|
357
|
+
this.updates.push({
|
|
358
|
+
updateOne: {
|
|
359
|
+
filter: { _id: bucketContext.docId(doc.o) },
|
|
360
|
+
update: {
|
|
361
|
+
$set: {
|
|
362
|
+
op: 'MOVE',
|
|
363
|
+
target_op: targetOp
|
|
364
|
+
},
|
|
365
|
+
$unset: {
|
|
366
|
+
source_table: 1,
|
|
367
|
+
source_key: 1,
|
|
368
|
+
subkey: 1,
|
|
369
|
+
table: 1,
|
|
370
|
+
row_id: 1,
|
|
371
|
+
data: 1
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
});
|
|
376
|
+
// TODO: better estimate for this.
|
|
377
|
+
currentState.opBytes += 200 - Number(size);
|
|
378
|
+
}
|
|
379
|
+
else if (currentState.trackingSize < idLimitBytes) {
|
|
380
|
+
// flatstr reduces the memory usage by flattening the string.
|
|
381
|
+
currentState.seen.set(utils.flatstr(key), doc.o);
|
|
382
|
+
// length + 16 for the string
|
|
383
|
+
// 24 for the bigint
|
|
384
|
+
// 50 for map overhead
|
|
385
|
+
// 50 for additional overhead
|
|
386
|
+
currentState.trackingSize += key.length + 140;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (isPersistentPut) {
|
|
390
|
+
currentState.lastNotPut = null;
|
|
391
|
+
currentState.opsSincePut = 0;
|
|
392
|
+
}
|
|
393
|
+
else if (doc.op != 'CLEAR') {
|
|
394
|
+
if (currentState.lastNotPut == null) {
|
|
395
|
+
currentState.lastNotPut = doc.o;
|
|
396
|
+
}
|
|
397
|
+
currentState.opsSincePut += 1;
|
|
398
|
+
}
|
|
399
|
+
if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
|
|
400
|
+
await this.flush(bucketContext);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
|
|
404
|
+
}
|
|
405
|
+
// Free memory before clearing the bucket.
|
|
406
|
+
currentState.seen.clear();
|
|
407
|
+
if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
|
|
408
|
+
this.logger.info(`Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
|
|
409
|
+
// Need flush() before clear().
|
|
410
|
+
await this.flush(bucketContext);
|
|
411
|
+
await this.clearBucket(currentState, bucketContext);
|
|
412
|
+
}
|
|
413
|
+
// Do this after clearBucket so we have accurate counts.
|
|
414
|
+
this.updateBucketChecksums(currentState, this.maxOpId);
|
|
415
|
+
// Need another flush after updateBucketChecksums().
|
|
416
|
+
await this.flush(bucketContext);
|
|
417
|
+
}
|
|
418
|
+
async flush(bucketContext) {
|
|
419
|
+
if (this.updates.length > 0) {
|
|
420
|
+
this.logger.info(`Compacting ${this.updates.length} ops`);
|
|
421
|
+
await bucketContext.collection.bulkWrite(this.updates, {
|
|
422
|
+
// Order is not important. Since checksums are not affected, these operations can happen in any order,
|
|
423
|
+
// and it's fine if the operations are partially applied. Each individual operation is atomic.
|
|
424
|
+
ordered: false
|
|
425
|
+
});
|
|
426
|
+
this.updates = [];
|
|
427
|
+
}
|
|
428
|
+
await this.flushBucketStateUpdates();
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Perform a CLEAR compact for a bucket.
|
|
432
|
+
*
|
|
433
|
+
* @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
|
|
434
|
+
*/
|
|
435
|
+
async clearBucket(currentState, bucketContext) {
|
|
436
|
+
const clearOp = currentState.lastNotPut;
|
|
437
|
+
const opFilter = {
|
|
438
|
+
_id: {
|
|
439
|
+
$gte: bucketContext.minId,
|
|
440
|
+
$lte: bucketContext.docId(clearOp)
|
|
441
|
+
}
|
|
442
|
+
};
|
|
443
|
+
const session = this.db.client.startSession();
|
|
444
|
+
try {
|
|
445
|
+
let done = false;
|
|
446
|
+
while (!done) {
|
|
447
|
+
this.signal?.throwIfAborted();
|
|
448
|
+
let opCountDiff = 0;
|
|
449
|
+
// Do the CLEAR operation in batches, with each batch a separate transaction.
|
|
450
|
+
// The state after each batch is fully consistent.
|
|
451
|
+
// We need a transaction per batch to make sure checksums stay consistent.
|
|
452
|
+
await session.withTransaction(async () => {
|
|
453
|
+
const query = bucketContext.collection.find(opFilter, {
|
|
454
|
+
session,
|
|
455
|
+
sort: { _id: 1 },
|
|
456
|
+
projection: {
|
|
457
|
+
_id: 1,
|
|
458
|
+
op: 1,
|
|
459
|
+
checksum: 1,
|
|
460
|
+
target_op: 1
|
|
461
|
+
},
|
|
462
|
+
limit: this.clearBatchLimit
|
|
463
|
+
});
|
|
464
|
+
let checksum = 0;
|
|
465
|
+
let lastOp = null;
|
|
466
|
+
let targetOp = null;
|
|
467
|
+
let gotAnOp = false;
|
|
468
|
+
let numberOfOpsToClear = 0;
|
|
469
|
+
for await (const rawOp of query.stream()) {
|
|
470
|
+
const op = bucketContext.fromPartialPersistedDocument(rawOp);
|
|
471
|
+
if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
|
|
472
|
+
checksum = utils.addChecksums(checksum, Number(op.checksum));
|
|
473
|
+
lastOp = op;
|
|
474
|
+
numberOfOpsToClear += 1;
|
|
475
|
+
if (op.op != 'CLEAR') {
|
|
476
|
+
gotAnOp = true;
|
|
477
|
+
}
|
|
478
|
+
if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
|
|
479
|
+
targetOp = op.target_op;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
else {
|
|
483
|
+
throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
if (!gotAnOp) {
|
|
487
|
+
done = true;
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
|
|
491
|
+
await bucketContext.collection.deleteMany({
|
|
492
|
+
_id: {
|
|
493
|
+
$gte: bucketContext.minId,
|
|
494
|
+
$lte: bucketContext.docId(lastOp.o)
|
|
495
|
+
}
|
|
496
|
+
}, { session });
|
|
497
|
+
const op = bucketContext.toPersistedDocument({
|
|
498
|
+
o: lastOp.o,
|
|
499
|
+
op: 'CLEAR',
|
|
500
|
+
checksum: BigInt(checksum),
|
|
501
|
+
data: null,
|
|
502
|
+
target_op: targetOp
|
|
503
|
+
});
|
|
504
|
+
await bucketContext.collection.insertOne(op, { session });
|
|
505
|
+
opCountDiff = -numberOfOpsToClear + 1;
|
|
506
|
+
}, {
|
|
507
|
+
writeConcern: { w: 'majority' },
|
|
508
|
+
readConcern: { level: 'snapshot' }
|
|
509
|
+
});
|
|
510
|
+
// Update outside the transaction, since the transaction can be retried multiple times.
|
|
511
|
+
currentState.opCount += opCountDiff;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
finally {
|
|
515
|
+
await session.endSession();
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
formatBucketDataKey(doc) {
|
|
519
|
+
return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
|
|
520
|
+
}
|
|
51
521
|
}
|
|
52
522
|
//# sourceMappingURL=MongoCompactorV1.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAI9E,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,aAAwC;QAExC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACrG,OAAO,EACL,YAAY,EAGZ,iBAAiB,EAEjB,aAAa,EACb,KAAK,EACN,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAuB,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAK/D,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AACrC,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAqB3C,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAcxC;IATF,OAAO,GAAwD,EAAE,CAAC;IAClE,kBAAkB,GAA2D,EAAE,CAAC;IACvE,gBAAgB,CAAS;IACzB,cAAc,CAAS;IACvB,OAAO,CAAS;IAEjC,YACE,aAAuC,EACvC,EAA6B,EACrB,OAA4B;QAEpC,KAAK,CAAC,aAAa,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QAF1B,YAAO,GAAP,OAAO,CAAqB;QAGpC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,0BAA0B,CAAC;QAC/E,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACzE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,OAAO;QACpB,IAAI,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC;YAClC,uBAAuB;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;YACrF,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,SAAS,CAAC,CAAC;YAChH,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEzC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClC,+FAA+F;gBAC/F,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,CACR,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CAAC;YACpD,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,+BAA+B,CAC5C,UAA0C,EAC1C,MAA2B,EAC3B,KAA0B,EAC1B,OAA6D;QAE7D,4EAA4E;QAC5E,OAAO,IAAI,EAAE,CAAC;YACZ,yEAAyE;YACzE,iEAAiE;YACjE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU;iBAC9B,SAAS,CACR;gBACE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE;gBAChD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,iFAAiF;gBACjF,EAAE,MAAM,EAAE,4BAA4B,EAAE;gBACxC;oBACE,MAAM,EAAE;wBACN,OAAO,EAAE;4BACP,EAAE,MAAM,EAAE,EAAE,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE,EAAE,EAAE;4BAClF,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,EAAE;yBACxE;wBACD,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;qBAC1E;iBACF;aACF,EACD,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAC1C;iBACA,OAAO,EAAE,CAAC;YAEb,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC;YAEpB,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;gBAC/D,sEAAsE;gBACtE,MAAM,YAAY,GAAG,WAAW,CAAC,sBAAsB,EAAE,KAAK,IAAI,CAAC,CAAC;gBACpE,MAAM,UAAU,GAAG,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC;gBAC5E,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,sBAAsB,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;gBAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC;gBAClF,OAAO;oBACL,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;oBACzB,YAAY,EAAE,IAAI;oBAClB,cAAc,EAAE,UAAU;oBAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,CAClB,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,EAC9C,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAC/C;iBACF,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,MAAM,CACvB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,UAAU,IAAI,OAAO,CAAC,cAAc,CAC7G,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yCAAyC,CACrD,MAA2C;QAE3C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa;aAC7C,IAAI,CAAC,MAAM,EAAE;YACZ,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE;YACrE,IAAI,EAAE,EAAE,8BAA8B,EAAE,CAAC,CAAC,EAAE;YAC5C,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,0BAA0B;SACtC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,YAAY,EAAE,IAAI;YAClB,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,sBAAuB,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC;SAC1G,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,MAAc,EAAE,gBAA2C,IAAI;QACtG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,IAAI,CAAC,kBAAkB,CAAC;YAClD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC,CAAC,EAAE,CAAC;YACH,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,KAAK,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,OAAO,EAAE,CAAC;gBAC/C,MAAM,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAAqC;QAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,qFAAqF;QACrF,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;YACjE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,4EAA4E;YAC5E,MAAM,YAAY,GAAmB,EAAE,CAAC;YACxC,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1B,kBAAkB,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC5C,IAAI,kBAAkB,GAAG,MAAM,EAAE,CAAC;oBAChC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sCAAsC,OAAO,CAAC,MAAM,gCAAgC,kBAAkB,EAAE,CACzG,CAAC;YACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,YAAY,CAAC,MAAM,eAAe,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;YAC7G,KAAK,IAAI,YAAY,CAAC,MAAM,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAEO,wBAAwB,CAC9B,KAAyB,EACzB,aAAqB;QAErB,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,OAAO;YACL,SAAS,EAAE;gBACT,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC;gBAC5C,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,aAAa;4BACpB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,8FAA8F;4BAC9F,mGAAmG;4BACnG,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;iBACoD;gBACvD,wDAAwD;gBACxD,0FAA0F;gBAC1F,MAAM,EAAE,KAAK;aACd;SACF,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAAC,KAAyB,EAAE,aAAqB;QAC5E,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,uBAAuB;QACnC,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YAC7E,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACrC,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAuD;QACxF,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEpG,KAAK,MAAM,cAAc,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,sDAAsD;gBACtD,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC3B,SAAS,EAAE;oBACT,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,MAAM,CAAC;oBACrD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,eAAe,EAAE;gCACf,KAAK,EAAE,IAAI,CAAC,OAAO;gCACnB,KAAK,EAAE,cAAc,CAAC,KAAK;gCAC3B,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;gCACzC,KAAK,EAAE,IAAI;6BACZ;4BACD,sBAAsB,EAAE;gCACtB,KAAK,EAAE,CAAC;gCACR,KAAK,EAAE,CAAC;6BACT;yBACF;qBACoD;oBACvD,mGAAmG;oBACnG,kBAAkB;oBAClB,MAAM,EAAE,KAAK;iBACd;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,EACH,EAAE,WAAW,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,CAC7C,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,MAAc;QACtC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,MAAc;QACzC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAChD,oEAAoE;QACpE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAE7B,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,YAAY,GAAuB;YACvC,MAAM;YACN,YAAY,EAAE,aAAa,CAAC,GAAG,CAAC,YAAY;YAC5C,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,CAAC;QAEF,wBAAwB;QACxB,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QACvC,0CAA0C;QAC1C,IAAI,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QAErC,oEAAoE;QACpE,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,uDAAuD;YACvD,MAAM,QAAQ,GAAG;gBACf;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,SAAS,CAAmD,QAAQ,EAAE;gBAC5G,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,mBAAmB,GAAG,CAAC;aACxC,CAAC,CAAC;YACH,+GAA+G;YAC/G,oCAAoC;YACpC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACtC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;gBACnC,OAAO;oBACL,GAAG,EAAE,aAAa,CAAC,qBAAqB,CAAC,IAAI,CAAC;oBAC9C,IAAI;iBACL,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,yBAAyB;gBACzB,MAAM;YACR,CAAC;YAED,sEAAsE;YACtE,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAE/C,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,+CAA+C;wBAC/C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gCAC3C,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,MAAM,EAAE,CAAC;wCACT,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACiD;6BACrD;yBACF,CAAC,CAAC;wBAEH,kCAAkC;wBAClC,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC7C,CAAC;yBAAM,IAAI,YAAY,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC;wBACpD,6DAA6D;wBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;wBACjD,6BAA6B;wBAC7B,oBAAoB;wBACpB,sBAAsB;wBACtB,6BAA6B;wBAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;oBAChD,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC;oBAClC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,MAAM,oBAAoB,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,0CAA0C;QAC1C,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sBAAsB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CAC5H,CAAC;YACF,+BAA+B;YAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACtD,CAAC;QAED,wDAAwD;QACxD,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,oDAAoD;QACpD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,aAAkC;QACpD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YAC1D,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBACrD,sGAAsG;gBACtG,8FAA8F;gBAC9F,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC,EAAE,aAAkC;QAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE,aAAa,CAAC,KAAK;gBACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;aACnC;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;gBAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CACzC,QAAQ,EACR;wBACE,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CACF,CAAC;oBACF,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,MAAM,GAA6D,IAAI,CAAC;oBAC5E,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACzC,MAAM,EAAE,GAAG,aAAa,CAAC,4BAA4B,CAAC,KAAK,CAAC,CAAC;wBAE7D,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,MAAM,GAAG,EAAE,CAAC;4BACZ,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;gCAC1E,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;4BAC1B,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC1G,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;oBACjF,MAAM,aAAa,CAAC,UAAU,CAAC,UAAU,CACvC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE,aAAa,CAAC,KAAK;4BACzB,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC;yBACrC;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;wBAC3C,CAAC,EAAE,MAAO,CAAC,CAAC;wBACZ,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,CAAC,CAAC;oBACH,MAAM,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;oBAE1D,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,uFAAuF;gBACvF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,GAA2C;QACrE,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,mBAAmB,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;IACjF,CAAC;CACF"}
|
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { bson, InternalOpId } from '@powersync/service-core';
|
|
2
3
|
import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
|
|
3
4
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
4
5
|
export declare class MongoParameterCompactorV1 extends MongoParameterCompactor {
|
|
5
6
|
protected readonly db: VersionedPowerSyncMongoV1;
|
|
6
7
|
protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
|
|
7
|
-
protected
|
|
8
|
-
|
|
8
|
+
protected shouldCompactDocument(doc: {
|
|
9
|
+
key: mongo.Document;
|
|
10
|
+
}): boolean;
|
|
11
|
+
/**
|
|
12
|
+
* Uses the legacy `{ 'key.g': 1, lookup: 1, _id: 1 }` index to narrow the stream, lookup and
|
|
13
|
+
* operation-id range. `key` is not part of that index at all, so it is a residual predicate applied
|
|
14
|
+
* to every document the range scan returns.
|
|
15
|
+
*
|
|
16
|
+
* That scan may therefore have to filter through many keys for the same lookup, but the cost is
|
|
17
|
+
* amortized: a single scan covers up to 1000 keys, and identities seen again in a later batch skip
|
|
18
|
+
* the scan entirely - they are deleted by `_id`.
|
|
19
|
+
*
|
|
20
|
+
* The V3 storage format uses an index more suitable for this.
|
|
21
|
+
*/
|
|
22
|
+
protected leadingHistoryDeleteFilter(lookup: bson.Binary, keys: mongo.Document[], before: InternalOpId): mongo.Document;
|
|
9
23
|
}
|
|
@@ -3,17 +3,28 @@ export class MongoParameterCompactorV1 extends MongoParameterCompactor {
|
|
|
3
3
|
async getCollections() {
|
|
4
4
|
return [this.db.parameterIndexV1];
|
|
5
5
|
}
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
// The shared V1 collection is scanned using only its default `_id` index. Filter the stream in
|
|
7
|
+
// code so compaction does not require an index on `key.g`.
|
|
8
|
+
shouldCompactDocument(doc) {
|
|
9
|
+
return doc.key.g === this.replicationStreamId;
|
|
10
10
|
}
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Uses the legacy `{ 'key.g': 1, lookup: 1, _id: 1 }` index to narrow the stream, lookup and
|
|
13
|
+
* operation-id range. `key` is not part of that index at all, so it is a residual predicate applied
|
|
14
|
+
* to every document the range scan returns.
|
|
15
|
+
*
|
|
16
|
+
* That scan may therefore have to filter through many keys for the same lookup, but the cost is
|
|
17
|
+
* amortized: a single scan covers up to 1000 keys, and identities seen again in a later batch skip
|
|
18
|
+
* the scan entirely - they are deleted by `_id`.
|
|
19
|
+
*
|
|
20
|
+
* The V3 storage format uses an index more suitable for this.
|
|
21
|
+
*/
|
|
22
|
+
leadingHistoryDeleteFilter(lookup, keys, before) {
|
|
12
23
|
return {
|
|
13
|
-
'key.g':
|
|
14
|
-
lookup
|
|
15
|
-
|
|
16
|
-
|
|
24
|
+
'key.g': this.replicationStreamId,
|
|
25
|
+
lookup,
|
|
26
|
+
key: { $in: keys },
|
|
27
|
+
_id: { $lt: before }
|
|
17
28
|
};
|
|
18
29
|
}
|
|
19
30
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,gBAA+D,CAAC,CAAC;IACnF,CAAC;IAED,+FAA+F;IAC/F,2DAA2D;IACxC,qBAAqB,CAAC,GAA4B;QACnE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,mBAAmB,CAAC;IAChD,CAAC;IAED;;;;;;;;;;OAUG;IACO,0BAA0B,CAClC,MAAmB,EACnB,IAAsB,EACtB,MAAoB;QAEpB,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,mBAAmB;YACjC,MAAM;YACN,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YAClB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE;SACrB,CAAC;IACJ,CAAC;CACF"}
|