@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
package/src/utils/util.ts
CHANGED
|
@@ -1,14 +1,56 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
|
-
import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { Logger, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
4
4
|
import { storage, utils } from '@powersync/service-core';
|
|
5
5
|
import * as bson from 'bson';
|
|
6
6
|
import * as crypto from 'crypto';
|
|
7
7
|
import * as timers from 'node:timers/promises';
|
|
8
8
|
import * as uuid from 'uuid';
|
|
9
9
|
import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
|
|
10
|
+
import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
|
|
10
11
|
|
|
11
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Default and max batch size for clearing.
|
|
14
|
+
*/
|
|
15
|
+
const CLEAR_BATCH_SIZE = 10_000;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Minimum batch size for clearing when a batch takes too long.
|
|
19
|
+
*/
|
|
20
|
+
const CLEAR_MIN_BATCH_SIZE = 100;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* We target batch duration to stay between the min and max. If the druation is outside this range,
|
|
24
|
+
* we increase or decrease the batch size.
|
|
25
|
+
*
|
|
26
|
+
* MONGO_CLEAR_OPERATION_TIMEOUT_MS controls the absolute max of individual find and delete operations,
|
|
27
|
+
* but we try to avoid hitting that.
|
|
28
|
+
*/
|
|
29
|
+
const CLEAR_BATCH_MIN_DURATION_MS = 500;
|
|
30
|
+
const CLEAR_BATCH_MAX_DURATION_MS = 2000;
|
|
31
|
+
|
|
32
|
+
type ClearCollectionFilter<T extends mongo.Document> = mongo.Filter<T> & {
|
|
33
|
+
_id: mongo.FilterOperators<mongo.InferIdType<T>>;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
function throwIfClearAborted(signal?: AbortSignal): void {
|
|
37
|
+
if (signal?.aborted) {
|
|
38
|
+
throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function waitWithSignal(delayMs: number, signal: AbortSignal | undefined, abortMessage: string): Promise<void> {
|
|
43
|
+
try {
|
|
44
|
+
await timers.setTimeout(delayMs, undefined, { signal });
|
|
45
|
+
} catch (error) {
|
|
46
|
+
if (signal?.aborted) {
|
|
47
|
+
throw new ReplicationAbortedError(abortMessage, signal.reason);
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.FilterOperators<T> {
|
|
12
54
|
let filter = {
|
|
13
55
|
$gte: {
|
|
14
56
|
...prefix
|
|
@@ -26,9 +68,9 @@ export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.
|
|
|
26
68
|
return filter;
|
|
27
69
|
}
|
|
28
70
|
|
|
29
|
-
export function
|
|
71
|
+
export function generateReplicationStreamName(prefix: string, replicationStreamId: number) {
|
|
30
72
|
const slot_suffix = crypto.randomBytes(2).toString('hex');
|
|
31
|
-
return `${prefix}${
|
|
73
|
+
return `${prefix}${replicationStreamId}_${slot_suffix}`;
|
|
32
74
|
}
|
|
33
75
|
|
|
34
76
|
/**
|
|
@@ -79,7 +121,9 @@ export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry {
|
|
|
79
121
|
object_type: row.table,
|
|
80
122
|
object_id: row.row_id,
|
|
81
123
|
checksum: Number(row.checksum),
|
|
82
|
-
|
|
124
|
+
// Storage V3 always persists this during replication. The fallback serves V1 data
|
|
125
|
+
// written before subkey persistence was introduced.
|
|
126
|
+
subkey: row.subkey ?? replicaIdToSubkey(row.source_table!, row.source_key!),
|
|
83
127
|
data: row.data
|
|
84
128
|
};
|
|
85
129
|
} else {
|
|
@@ -141,6 +185,7 @@ export async function retryOnMongoMaxTimeMSExpired<T>(
|
|
|
141
185
|
}
|
|
142
186
|
): Promise<T> {
|
|
143
187
|
let retryCount = 0;
|
|
188
|
+
// Retry indefinitely on MaxTimeMSExpired errors with exponential backoff.
|
|
144
189
|
while (true) {
|
|
145
190
|
if (options.signal?.aborted) {
|
|
146
191
|
throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
|
|
@@ -148,16 +193,152 @@ export async function retryOnMongoMaxTimeMSExpired<T>(
|
|
|
148
193
|
try {
|
|
149
194
|
return await operation();
|
|
150
195
|
} catch (e) {
|
|
151
|
-
if (!lib_mongo.
|
|
196
|
+
if (!lib_mongo.isMaxTimeMSExpiredError(e)) {
|
|
152
197
|
throw e;
|
|
153
198
|
}
|
|
154
199
|
retryCount += 1;
|
|
155
200
|
options.onRetry?.(retryCount);
|
|
156
|
-
await
|
|
201
|
+
await waitWithSignal(options.retryDelayMs, options.signal, options.abortMessage ?? 'Aborted MongoDB operation');
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function clearCollectionInBatches(
|
|
207
|
+
logger: Logger,
|
|
208
|
+
label: string,
|
|
209
|
+
findAndDeleteBatch: (batchSize: number) => Promise<number>,
|
|
210
|
+
signal?: AbortSignal,
|
|
211
|
+
throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
|
|
212
|
+
): Promise<number> {
|
|
213
|
+
let batchSize = CLEAR_BATCH_SIZE;
|
|
214
|
+
let deletedCount = 0;
|
|
215
|
+
while (true) {
|
|
216
|
+
throwIfClearAborted(signal);
|
|
217
|
+
try {
|
|
218
|
+
const batchStartedAt = performance.now();
|
|
219
|
+
const foundBatchSize = await findAndDeleteBatch(batchSize);
|
|
220
|
+
const batchDurationMs = performance.now() - batchStartedAt;
|
|
221
|
+
deletedCount += foundBatchSize;
|
|
222
|
+
if (foundBatchSize > 0) {
|
|
223
|
+
logger.info(
|
|
224
|
+
`Cleared batch of ${label} (${foundBatchSize} documents) in ${Math.round(batchDurationMs)}ms, continuing...`
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
if (foundBatchSize < batchSize) {
|
|
228
|
+
return deletedCount;
|
|
229
|
+
}
|
|
230
|
+
if (batchDurationMs > CLEAR_BATCH_MAX_DURATION_MS) {
|
|
231
|
+
batchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
|
|
232
|
+
} else if (batchDurationMs < CLEAR_BATCH_MIN_DURATION_MS) {
|
|
233
|
+
batchSize = Math.min(CLEAR_BATCH_SIZE, batchSize * 2);
|
|
234
|
+
}
|
|
235
|
+
await waitWithSignal(batchDurationMs * throttleRate, signal, 'Aborted clearing data');
|
|
236
|
+
} catch (error) {
|
|
237
|
+
if (!lib_mongo.isTimeoutError(error) || batchSize == CLEAR_MIN_BATCH_SIZE) {
|
|
238
|
+
throw error;
|
|
239
|
+
}
|
|
240
|
+
const nextBatchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
|
|
241
|
+
logger.info(
|
|
242
|
+
`Clearing batch of ${label} timed out with batch size ${batchSize}, retrying find and delete with ${nextBatchSize}...`
|
|
243
|
+
);
|
|
244
|
+
batchSize = nextBatchSize;
|
|
245
|
+
|
|
246
|
+
// Pause for at least the equivalent of one query timeout before retrying, since the cause may be high database load.
|
|
247
|
+
const pauseDuration = Math.max(throttleRate, 1.0) * lib_mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS;
|
|
248
|
+
await waitWithSignal(pauseDuration, signal, 'Aborted clearing data');
|
|
157
249
|
}
|
|
158
250
|
}
|
|
159
251
|
}
|
|
160
252
|
|
|
253
|
+
export async function clearCollectionInIdRanges<T extends mongo.Document>(
|
|
254
|
+
logger: Logger,
|
|
255
|
+
label: string,
|
|
256
|
+
collection: mongo.Collection<T>,
|
|
257
|
+
filter: ClearCollectionFilter<T>,
|
|
258
|
+
signal?: AbortSignal,
|
|
259
|
+
throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
|
|
260
|
+
): Promise<number> {
|
|
261
|
+
return clearCollectionInBatches(
|
|
262
|
+
logger,
|
|
263
|
+
label,
|
|
264
|
+
async (batchSize) => {
|
|
265
|
+
const queryOptions: mongo.FindOptions<T> = {
|
|
266
|
+
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
|
|
267
|
+
projection: { _id: 1 }
|
|
268
|
+
};
|
|
269
|
+
const [batchEnd] = await collection
|
|
270
|
+
.find(filter, queryOptions)
|
|
271
|
+
.sort({ _id: 1 })
|
|
272
|
+
.skip(batchSize)
|
|
273
|
+
.limit(1)
|
|
274
|
+
.toArray();
|
|
275
|
+
throwIfClearAborted(signal);
|
|
276
|
+
|
|
277
|
+
if (batchEnd == null) {
|
|
278
|
+
// We're on the last batch
|
|
279
|
+
const result = await collection.deleteMany(filter, {
|
|
280
|
+
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
|
|
281
|
+
});
|
|
282
|
+
return result.deletedCount;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
const idRange: mongo.FilterOperators<mongo.InferIdType<T>> = {
|
|
286
|
+
...filter._id,
|
|
287
|
+
$lt: batchEnd._id
|
|
288
|
+
};
|
|
289
|
+
await collection.deleteMany(
|
|
290
|
+
{
|
|
291
|
+
...filter,
|
|
292
|
+
_id: idRange
|
|
293
|
+
} as mongo.Filter<T>,
|
|
294
|
+
{ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
295
|
+
);
|
|
296
|
+
return batchSize;
|
|
297
|
+
},
|
|
298
|
+
signal,
|
|
299
|
+
throttleRate
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export async function clearCollectionInIdBatches<T extends mongo.Document>(
|
|
304
|
+
logger: Logger,
|
|
305
|
+
label: string,
|
|
306
|
+
collection: mongo.Collection<T>,
|
|
307
|
+
filter: mongo.Filter<T>,
|
|
308
|
+
signal?: AbortSignal,
|
|
309
|
+
throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
|
|
310
|
+
): Promise<number> {
|
|
311
|
+
return clearCollectionInBatches(
|
|
312
|
+
logger,
|
|
313
|
+
label,
|
|
314
|
+
async (batchSize) => {
|
|
315
|
+
const documents = await collection
|
|
316
|
+
.find(filter, {
|
|
317
|
+
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
|
|
318
|
+
projection: { _id: 1 }
|
|
319
|
+
})
|
|
320
|
+
.limit(batchSize)
|
|
321
|
+
.toArray();
|
|
322
|
+
throwIfClearAborted(signal);
|
|
323
|
+
|
|
324
|
+
if (documents.length === 0) {
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
await collection.deleteMany(
|
|
328
|
+
{
|
|
329
|
+
_id: {
|
|
330
|
+
$in: documents.map((document) => document._id)
|
|
331
|
+
}
|
|
332
|
+
} as mongo.Filter<T>,
|
|
333
|
+
{ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
334
|
+
);
|
|
335
|
+
return documents.length;
|
|
336
|
+
},
|
|
337
|
+
signal,
|
|
338
|
+
throttleRate
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
|
|
161
342
|
export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
|
|
162
343
|
query: mongo.Filter<T>,
|
|
163
344
|
collection: mongo.Collection<T>,
|
|
@@ -1,56 +1,6 @@
|
|
|
1
1
|
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
|
2
2
|
|
|
3
|
-
exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
|
|
4
|
-
[
|
|
5
|
-
{
|
|
6
|
-
"checksum": 2871785649,
|
|
7
|
-
"object_id": "test1",
|
|
8
|
-
"op": "PUT",
|
|
9
|
-
},
|
|
10
|
-
{
|
|
11
|
-
"checksum": 2872534815,
|
|
12
|
-
"object_id": "test1",
|
|
13
|
-
"op": "REMOVE",
|
|
14
|
-
},
|
|
15
|
-
{
|
|
16
|
-
"checksum": 2871785649,
|
|
17
|
-
"object_id": "test1",
|
|
18
|
-
"op": "PUT",
|
|
19
|
-
},
|
|
20
|
-
{
|
|
21
|
-
"checksum": 2872534815,
|
|
22
|
-
"object_id": "test1",
|
|
23
|
-
"op": "REMOVE",
|
|
24
|
-
},
|
|
25
|
-
]
|
|
26
|
-
`;
|
|
27
|
-
|
|
28
|
-
exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
|
|
29
|
-
[
|
|
30
|
-
{
|
|
31
|
-
"checksum": 2871785649,
|
|
32
|
-
"object_id": "test1",
|
|
33
|
-
"op": "PUT",
|
|
34
|
-
},
|
|
35
|
-
{
|
|
36
|
-
"checksum": 2872534815,
|
|
37
|
-
"object_id": "test1",
|
|
38
|
-
"op": "REMOVE",
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
"checksum": 2871785649,
|
|
42
|
-
"object_id": "test1",
|
|
43
|
-
"op": "PUT",
|
|
44
|
-
},
|
|
45
|
-
{
|
|
46
|
-
"checksum": 2872534815,
|
|
47
|
-
"object_id": "test1",
|
|
48
|
-
"op": "REMOVE",
|
|
49
|
-
},
|
|
50
|
-
]
|
|
51
|
-
`;
|
|
52
|
-
|
|
53
|
-
exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
|
|
3
|
+
exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
|
|
54
4
|
[
|
|
55
5
|
{
|
|
56
6
|
"checksum": 2871785649,
|
|
@@ -75,7 +25,7 @@ exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3
|
|
|
75
25
|
]
|
|
76
26
|
`;
|
|
77
27
|
|
|
78
|
-
exports[`Mongo Sync Bucket Storage - Data -
|
|
28
|
+
exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
|
|
79
29
|
[
|
|
80
30
|
{
|
|
81
31
|
"checksum": 2871785649,
|
|
@@ -100,7 +50,7 @@ exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (dele
|
|
|
100
50
|
]
|
|
101
51
|
`;
|
|
102
52
|
|
|
103
|
-
exports[`Mongo Sync Bucket Storage - Data -
|
|
53
|
+
exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
|
|
104
54
|
[
|
|
105
55
|
{
|
|
106
56
|
"checksum": 2871785649,
|
|
@@ -125,7 +75,7 @@ exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (dele
|
|
|
125
75
|
]
|
|
126
76
|
`;
|
|
127
77
|
|
|
128
|
-
exports[`Mongo Sync Bucket Storage - Data -
|
|
78
|
+
exports[`Mongo Sync Bucket Storage - Data - v4 > (insert, delete, insert), (delete) 1`] = `
|
|
129
79
|
[
|
|
130
80
|
{
|
|
131
81
|
"checksum": 2871785649,
|