@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import * as timers from 'node:timers/promises';
|
|
2
|
+
|
|
1
3
|
import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
|
|
2
4
|
import {
|
|
3
5
|
logger as defaultLogger,
|
|
@@ -5,24 +7,16 @@ import {
|
|
|
5
7
|
ReplicationAssertionError,
|
|
6
8
|
ServiceAssertionError
|
|
7
9
|
} from '@powersync/lib-services-framework';
|
|
8
|
-
import {
|
|
9
|
-
addChecksums,
|
|
10
|
-
InternalOpId,
|
|
11
|
-
isPartialChecksum,
|
|
12
|
-
PopulateChecksumCacheResults,
|
|
13
|
-
storage,
|
|
14
|
-
utils
|
|
15
|
-
} from '@powersync/service-core';
|
|
10
|
+
import { InternalOpId, isPartialChecksum, PopulateChecksumCacheResults, storage } from '@powersync/service-core';
|
|
16
11
|
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
17
12
|
|
|
18
|
-
import {
|
|
19
|
-
import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js';
|
|
13
|
+
import { BucketKey } from './common/BucketDataDoc.js';
|
|
20
14
|
import type { VersionedPowerSyncMongo } from './db.js';
|
|
21
15
|
import { BucketStateDocumentBase } from './models.js';
|
|
22
16
|
import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
|
|
23
|
-
import {
|
|
17
|
+
import { isRetryableObjectStorageError } from './v3/object-storage/ObjectStorage.js';
|
|
24
18
|
|
|
25
|
-
interface CurrentBucketState {
|
|
19
|
+
export interface CurrentBucketState {
|
|
26
20
|
/** Bucket name */
|
|
27
21
|
bucket: string;
|
|
28
22
|
definitionId: BucketDefinitionId;
|
|
@@ -56,18 +50,26 @@ interface CurrentBucketState {
|
|
|
56
50
|
opBytes: number;
|
|
57
51
|
}
|
|
58
52
|
|
|
59
|
-
type CompactClearProperties = 'op' | 'checksum' | 'target_op';
|
|
60
|
-
|
|
61
53
|
export interface MongoCompactOptions extends storage.CompactOptions {}
|
|
62
54
|
|
|
63
55
|
const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
|
|
64
56
|
const DEFAULT_MOVE_BATCH_LIMIT = 2000;
|
|
65
57
|
const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
|
|
58
|
+
const DEFAULT_MOVE_BATCH_BYTE_LIMIT = 16 * 1024 * 1024;
|
|
66
59
|
const DEFAULT_MIN_BUCKET_CHANGES = 10;
|
|
67
60
|
const DEFAULT_MIN_CHANGE_RATIO = 0.1;
|
|
68
61
|
const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
|
|
69
62
|
/** This default is primarily for tests. */
|
|
70
63
|
const DEFAULT_MEMORY_LIMIT_MB = 64;
|
|
64
|
+
const COMPACTION_RETRY_LIMIT = 3;
|
|
65
|
+
const COMPACTION_RETRY_DELAY_MS = 1_000;
|
|
66
|
+
|
|
67
|
+
export class ConcurrentCompactionError extends Error {
|
|
68
|
+
constructor(message: string) {
|
|
69
|
+
super(message);
|
|
70
|
+
this.name = 'ConcurrentCompactionError';
|
|
71
|
+
}
|
|
72
|
+
}
|
|
71
73
|
|
|
72
74
|
export interface DirtyBucket {
|
|
73
75
|
bucket: string;
|
|
@@ -77,17 +79,18 @@ export interface DirtyBucket {
|
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
export abstract class MongoCompactor {
|
|
80
|
-
protected updates: mongo.AnyBulkWriteOperation<BucketDataDocumentGeneric>[] = [];
|
|
81
82
|
protected bucketStateUpdates: mongo.AnyBulkWriteOperation<BucketStateDocumentBase>[] = [];
|
|
82
83
|
|
|
83
84
|
protected readonly idLimitBytes: number;
|
|
84
85
|
protected readonly moveBatchLimit: number;
|
|
85
86
|
protected readonly moveBatchQueryLimit: number;
|
|
87
|
+
protected readonly moveBatchByteLimit: number;
|
|
86
88
|
protected readonly clearBatchLimit: number;
|
|
87
89
|
protected readonly minBucketChanges: number;
|
|
88
90
|
protected readonly minChangeRatio: number;
|
|
89
91
|
protected readonly maxOpId: bigint;
|
|
90
92
|
protected readonly buckets: string[] | undefined;
|
|
93
|
+
protected readonly deleteCheckpointRequestsBefore: Date | undefined;
|
|
91
94
|
protected readonly signal?: AbortSignal;
|
|
92
95
|
protected readonly group_id: number;
|
|
93
96
|
|
|
@@ -98,15 +101,20 @@ export abstract class MongoCompactor {
|
|
|
98
101
|
protected readonly db: VersionedPowerSyncMongo,
|
|
99
102
|
options: MongoCompactOptions
|
|
100
103
|
) {
|
|
101
|
-
this.group_id = storage.
|
|
104
|
+
this.group_id = storage.replicationStreamId;
|
|
102
105
|
this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
|
|
103
106
|
this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
|
|
104
107
|
this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
|
|
108
|
+
this.moveBatchByteLimit = options.moveBatchByteLimit ?? DEFAULT_MOVE_BATCH_BYTE_LIMIT;
|
|
105
109
|
this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
110
|
+
if (this.clearBatchLimit < 2) {
|
|
111
|
+
throw new ReplicationAssertionError('clearBatchLimit must be >= 2');
|
|
112
|
+
}
|
|
106
113
|
this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
|
|
107
114
|
this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
|
|
108
115
|
this.maxOpId = options.maxOpId ?? 0n;
|
|
109
116
|
this.buckets = options.compactBuckets;
|
|
117
|
+
this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
|
|
110
118
|
this.signal = options.signal;
|
|
111
119
|
this.logger = options.logger ?? defaultLogger;
|
|
112
120
|
}
|
|
@@ -114,9 +122,11 @@ export abstract class MongoCompactor {
|
|
|
114
122
|
/**
|
|
115
123
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
116
124
|
*
|
|
117
|
-
* See /docs/compacting-operations.md for details.
|
|
125
|
+
* See /docs/storage/compacting-operations.md for details.
|
|
118
126
|
*/
|
|
119
127
|
async compact() {
|
|
128
|
+
await this.deleteOldCheckpointRequests();
|
|
129
|
+
|
|
120
130
|
if (this.buckets) {
|
|
121
131
|
for (const bucket of this.buckets) {
|
|
122
132
|
// We can make this more efficient later on by iterating through the buckets in a single query.
|
|
@@ -128,11 +138,31 @@ export abstract class MongoCompactor {
|
|
|
128
138
|
}
|
|
129
139
|
}
|
|
130
140
|
|
|
141
|
+
private async deleteOldCheckpointRequests() {
|
|
142
|
+
if (this.deleteCheckpointRequestsBefore == null) {
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
this.signal?.throwIfAborted();
|
|
147
|
+
// The explicit $exists guarantees the query is a subset of the
|
|
148
|
+
// checkpoint_requested_at partial index's filter, so the planner can use it.
|
|
149
|
+
// Keep pending requests during replication lag so a retry doesn't advance the associated
|
|
150
|
+
// LSN even further.
|
|
151
|
+
await this.db.write_checkpoints.deleteMany({
|
|
152
|
+
checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore },
|
|
153
|
+
processed_at_lsn: { $ne: null }
|
|
154
|
+
});
|
|
155
|
+
await this.db.custom_write_checkpoints.deleteMany({
|
|
156
|
+
checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore }
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
131
160
|
/**
|
|
132
161
|
* Subset of compact, only populating checksums where relevant.
|
|
133
162
|
*/
|
|
134
163
|
async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
|
|
135
164
|
let count = 0;
|
|
165
|
+
// Paginate through dirty buckets in batches until no more buckets meet the criteria.
|
|
136
166
|
while (true) {
|
|
137
167
|
this.signal?.throwIfAborted();
|
|
138
168
|
const buckets = await this.dirtyBucketBatchForChecksums(options);
|
|
@@ -172,6 +202,7 @@ export abstract class MongoCompactor {
|
|
|
172
202
|
},
|
|
173
203
|
getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null
|
|
174
204
|
): AsyncGenerator<DirtyBucket[]> {
|
|
205
|
+
// Paginate through the bucket state collection using cursor-based scanning.
|
|
175
206
|
while (true) {
|
|
176
207
|
// To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline
|
|
177
208
|
// to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria.
|
|
@@ -300,207 +331,66 @@ export abstract class MongoCompactor {
|
|
|
300
331
|
/**
|
|
301
332
|
* Compaction for a single bucket, with retries on failure.
|
|
302
333
|
*
|
|
303
|
-
*
|
|
334
|
+
* A compaction can race another compactor after its initial scan. Restarting
|
|
335
|
+
* the bucket is safe because each replacement is transactional. Object
|
|
336
|
+
* storage paths are also prepared with lifecycle markers, so a retry can
|
|
337
|
+
* safely overwrite or eventually clean up uploads from the failed attempt.
|
|
304
338
|
*/
|
|
305
339
|
protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) {
|
|
306
340
|
let retryCount = 0;
|
|
307
341
|
while (true) {
|
|
342
|
+
this.signal?.throwIfAborted();
|
|
343
|
+
// Do not carry queued bucket state writes from a failed attempt into the rescan.
|
|
344
|
+
this.bucketStateUpdates = [];
|
|
308
345
|
try {
|
|
309
346
|
await this.compactSingleBucket(bucket, definitionId);
|
|
310
|
-
|
|
347
|
+
return;
|
|
311
348
|
} catch (e) {
|
|
312
|
-
if (
|
|
313
|
-
this.logger.warn(`Error compacting bucket ${bucket}, retrying...`, e);
|
|
314
|
-
retryCount++;
|
|
315
|
-
await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount));
|
|
316
|
-
} else {
|
|
349
|
+
if (this.signal?.aborted) {
|
|
317
350
|
throw e;
|
|
318
351
|
}
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) {
|
|
324
|
-
const idLimitBytes = this.idLimitBytes;
|
|
325
|
-
const bucketContext = await this.getBucketDataContext(bucket, definitionId);
|
|
326
|
-
if (bucketContext == null) {
|
|
327
|
-
return;
|
|
328
|
-
}
|
|
329
|
-
const currentState: CurrentBucketState = {
|
|
330
|
-
bucket,
|
|
331
|
-
definitionId: bucketContext.key.definitionId,
|
|
332
|
-
seen: new Map(),
|
|
333
|
-
trackingSize: 0,
|
|
334
|
-
lastNotPut: null,
|
|
335
|
-
opsSincePut: 0,
|
|
336
|
-
checksum: 0,
|
|
337
|
-
opCount: 0,
|
|
338
|
-
opBytes: 0
|
|
339
|
-
};
|
|
340
352
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
let upperBound = bucketContext.maxId;
|
|
345
|
-
|
|
346
|
-
while (true) {
|
|
347
|
-
this.signal?.throwIfAborted();
|
|
348
|
-
|
|
349
|
-
// Query one batch at a time, to avoid cursor timeouts.
|
|
350
|
-
const pipeline = [
|
|
351
|
-
{
|
|
352
|
-
$match: {
|
|
353
|
-
_id: {
|
|
354
|
-
$gte: lowerBound,
|
|
355
|
-
$lt: upperBound
|
|
356
|
-
},
|
|
357
|
-
// Workaround for a clustered collection bug where the $lt operator may include upperBound.
|
|
358
|
-
// Technically only needed for storage V3.
|
|
359
|
-
// https://jira.mongodb.org/browse/SERVER-121822
|
|
360
|
-
'_id.o': { $lt: upperBound.o }
|
|
361
|
-
}
|
|
362
|
-
},
|
|
363
|
-
{ $sort: { _id: -1 } },
|
|
364
|
-
{ $limit: this.moveBatchQueryLimit },
|
|
365
|
-
{
|
|
366
|
-
$project: {
|
|
367
|
-
_id: 1,
|
|
368
|
-
op: 1,
|
|
369
|
-
table: 1,
|
|
370
|
-
row_id: 1,
|
|
371
|
-
source_table: 1,
|
|
372
|
-
source_key: 1,
|
|
373
|
-
checksum: 1,
|
|
374
|
-
size: { $bsonSize: '$$ROOT' }
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
];
|
|
378
|
-
|
|
379
|
-
const cursor = bucketContext.collection.aggregate<BucketDataDocumentGeneric & { size: number | bigint }>(
|
|
380
|
-
pipeline,
|
|
381
|
-
{
|
|
382
|
-
// batchSize is 1 more than limit to auto-close the cursor.
|
|
383
|
-
// See https://github.com/mongodb/node-mongodb-native/pull/4580
|
|
384
|
-
batchSize: this.moveBatchQueryLimit + 1
|
|
385
|
-
}
|
|
386
|
-
);
|
|
387
|
-
// We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
|
|
388
|
-
// Instead, we load up to the limit.
|
|
389
|
-
const rawBatch = await cursor.toArray();
|
|
390
|
-
const batch = rawBatch.map((document) => {
|
|
391
|
-
const { size, ...rest } = document;
|
|
392
|
-
return {
|
|
393
|
-
doc: bucketContext.fromPersistedDocument(rest),
|
|
394
|
-
size
|
|
395
|
-
};
|
|
396
|
-
});
|
|
397
|
-
|
|
398
|
-
if (batch.length == 0) {
|
|
399
|
-
// We've reached the end.
|
|
400
|
-
break;
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
// Reuse the exact collection _id value from Mongo for the next bound.
|
|
404
|
-
upperBound = rawBatch[rawBatch.length - 1]._id;
|
|
405
|
-
|
|
406
|
-
for (const { doc, size } of batch) {
|
|
407
|
-
if (doc.o > this.maxOpId) {
|
|
408
|
-
continue;
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
|
|
412
|
-
currentState.opCount += 1;
|
|
413
|
-
|
|
414
|
-
let isPersistentPut = doc.op == 'PUT';
|
|
415
|
-
|
|
416
|
-
currentState.opBytes += Number(size);
|
|
417
|
-
if (doc.op == 'REMOVE' || doc.op == 'PUT') {
|
|
418
|
-
const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`;
|
|
419
|
-
const targetOp = currentState.seen.get(key);
|
|
420
|
-
if (targetOp) {
|
|
421
|
-
// Will convert to MOVE, so don't count as PUT.
|
|
422
|
-
isPersistentPut = false;
|
|
423
|
-
|
|
424
|
-
this.updates.push({
|
|
425
|
-
updateOne: {
|
|
426
|
-
filter: { _id: bucketContext.docId(doc.o) },
|
|
427
|
-
update: {
|
|
428
|
-
$set: {
|
|
429
|
-
op: 'MOVE',
|
|
430
|
-
target_op: targetOp
|
|
431
|
-
},
|
|
432
|
-
$unset: {
|
|
433
|
-
source_table: 1,
|
|
434
|
-
source_key: 1,
|
|
435
|
-
table: 1,
|
|
436
|
-
row_id: 1,
|
|
437
|
-
data: 1
|
|
438
|
-
}
|
|
439
|
-
} satisfies mongo.UpdateFilter<BucketDataDocumentGeneric>
|
|
440
|
-
}
|
|
441
|
-
});
|
|
442
|
-
|
|
443
|
-
// TODO: better estimate for this.
|
|
444
|
-
currentState.opBytes += 200 - Number(size);
|
|
445
|
-
} else if (currentState.trackingSize < idLimitBytes) {
|
|
446
|
-
// flatstr reduces the memory usage by flattening the string.
|
|
447
|
-
currentState.seen.set(utils.flatstr(key), doc.o);
|
|
448
|
-
// length + 16 for the string
|
|
449
|
-
// 24 for the bigint
|
|
450
|
-
// 50 for map overhead
|
|
451
|
-
// 50 for additional overhead
|
|
452
|
-
currentState.trackingSize += key.length + 140;
|
|
453
|
-
}
|
|
353
|
+
const retryReason = compactionRetryReason(e);
|
|
354
|
+
if (retryReason == null) {
|
|
355
|
+
throw e;
|
|
454
356
|
}
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
if (currentState.lastNotPut == null) {
|
|
461
|
-
currentState.lastNotPut = doc.o;
|
|
462
|
-
}
|
|
463
|
-
currentState.opsSincePut += 1;
|
|
357
|
+
if (retryCount >= COMPACTION_RETRY_LIMIT) {
|
|
358
|
+
throw new Error(
|
|
359
|
+
`Failed to compact bucket ${bucket} after ${retryCount + 1} attempts (${retryReason}): ${errorMessage(e)}`,
|
|
360
|
+
{ cause: e }
|
|
361
|
+
);
|
|
464
362
|
}
|
|
465
363
|
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
364
|
+
retryCount++;
|
|
365
|
+
const delay = COMPACTION_RETRY_DELAY_MS * 2 ** (retryCount - 1);
|
|
366
|
+
this.logger.warn(
|
|
367
|
+
`Error compacting bucket ${bucket} (${retryReason}); retrying in ${delay}ms ` +
|
|
368
|
+
`(attempt ${retryCount + 1}/${COMPACTION_RETRY_LIMIT + 1})`,
|
|
369
|
+
e
|
|
370
|
+
);
|
|
371
|
+
await timers.setTimeout(delay, undefined, this.signal ? { signal: this.signal } : undefined);
|
|
469
372
|
}
|
|
470
|
-
|
|
471
|
-
this.logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
// Free memory before clearing the bucket.
|
|
475
|
-
currentState.seen.clear();
|
|
476
|
-
if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
|
|
477
|
-
this.logger.info(
|
|
478
|
-
`Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
|
|
479
|
-
);
|
|
480
|
-
// Need flush() before clear().
|
|
481
|
-
await this.flush(bucketContext);
|
|
482
|
-
await this.clearBucket(currentState, bucketContext);
|
|
483
373
|
}
|
|
484
|
-
|
|
485
|
-
// Do this after clearBucket so we have accurate counts.
|
|
486
|
-
this.updateBucketChecksums(currentState);
|
|
487
|
-
// Need another flush after updateBucketChecksums().
|
|
488
|
-
await this.flush(bucketContext);
|
|
489
374
|
}
|
|
490
375
|
|
|
491
|
-
protected
|
|
376
|
+
protected abstract compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
|
|
377
|
+
|
|
378
|
+
protected collectBucketStateUpdates(
|
|
379
|
+
state: CurrentBucketState,
|
|
380
|
+
compactedOpId: InternalOpId
|
|
381
|
+
): mongo.AnyBulkWriteOperation<BucketStateDocumentBase> {
|
|
492
382
|
if (state.opCount < 0) {
|
|
493
383
|
throw new ServiceAssertionError(
|
|
494
384
|
`Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`
|
|
495
385
|
);
|
|
496
386
|
}
|
|
497
|
-
|
|
387
|
+
return {
|
|
498
388
|
updateOne: {
|
|
499
389
|
filter: this.bucketStateFilter(state.bucket, state.definitionId),
|
|
500
390
|
update: {
|
|
501
391
|
$set: {
|
|
502
392
|
compacted_state: {
|
|
503
|
-
op_id:
|
|
393
|
+
op_id: compactedOpId,
|
|
504
394
|
count: state.opCount,
|
|
505
395
|
checksum: BigInt(state.checksum),
|
|
506
396
|
bytes: state.opBytes
|
|
@@ -517,24 +407,14 @@ export abstract class MongoCompactor {
|
|
|
517
407
|
// We don't create new ones here, to avoid issues with the unique index on bucket_updates.
|
|
518
408
|
upsert: false
|
|
519
409
|
}
|
|
520
|
-
}
|
|
410
|
+
};
|
|
521
411
|
}
|
|
522
412
|
|
|
523
|
-
protected
|
|
524
|
-
|
|
525
|
-
this.logger.info(`Compacting ${this.updates.length} ops`);
|
|
526
|
-
await col.collection.bulkWrite(this.updates, {
|
|
527
|
-
// Order is not important. Since checksums are not affected, these operations can happen in any order,
|
|
528
|
-
// and it's fine if the operations are partially applied. Each individual operation is atomic.
|
|
529
|
-
ordered: false
|
|
530
|
-
});
|
|
531
|
-
this.updates = [];
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
await this.flushBucketStateUpdates();
|
|
413
|
+
protected updateBucketChecksums(state: CurrentBucketState, compactedOpId: InternalOpId) {
|
|
414
|
+
this.bucketStateUpdates.push(this.collectBucketStateUpdates(state, compactedOpId));
|
|
535
415
|
}
|
|
536
416
|
|
|
537
|
-
|
|
417
|
+
protected async flushBucketStateUpdates() {
|
|
538
418
|
if (this.bucketStateUpdates.length > 0) {
|
|
539
419
|
this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
|
|
540
420
|
await this.writeBucketStateUpdates();
|
|
@@ -542,108 +422,6 @@ export abstract class MongoCompactor {
|
|
|
542
422
|
}
|
|
543
423
|
}
|
|
544
424
|
|
|
545
|
-
/**
|
|
546
|
-
* Perform a CLEAR compact for a bucket.
|
|
547
|
-
*
|
|
548
|
-
* @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
|
|
549
|
-
*/
|
|
550
|
-
protected async clearBucket(currentState: CurrentBucketState, col: SingleBucketStore) {
|
|
551
|
-
const clearOp = currentState.lastNotPut!;
|
|
552
|
-
|
|
553
|
-
const opFilter = {
|
|
554
|
-
_id: {
|
|
555
|
-
$gte: col.minId,
|
|
556
|
-
$lte: col.docId(clearOp)
|
|
557
|
-
}
|
|
558
|
-
};
|
|
559
|
-
|
|
560
|
-
const session = this.db.client.startSession();
|
|
561
|
-
try {
|
|
562
|
-
let done = false;
|
|
563
|
-
while (!done) {
|
|
564
|
-
this.signal?.throwIfAborted();
|
|
565
|
-
let opCountDiff = 0;
|
|
566
|
-
// Do the CLEAR operation in batches, with each batch a separate transaction.
|
|
567
|
-
// The state after each batch is fully consistent.
|
|
568
|
-
// We need a transaction per batch to make sure checksums stay consistent.
|
|
569
|
-
await session.withTransaction(
|
|
570
|
-
async () => {
|
|
571
|
-
const query = col.collection.find<Pick<BucketDataDocumentGeneric, '_id' | CompactClearProperties>>(
|
|
572
|
-
opFilter,
|
|
573
|
-
{
|
|
574
|
-
session,
|
|
575
|
-
sort: { _id: 1 },
|
|
576
|
-
projection: {
|
|
577
|
-
_id: 1,
|
|
578
|
-
op: 1,
|
|
579
|
-
checksum: 1,
|
|
580
|
-
target_op: 1
|
|
581
|
-
},
|
|
582
|
-
limit: this.clearBatchLimit
|
|
583
|
-
}
|
|
584
|
-
);
|
|
585
|
-
let checksum = 0;
|
|
586
|
-
let lastOp: Pick<BucketDataDoc, 'o' | CompactClearProperties> | null = null;
|
|
587
|
-
let targetOp: bigint | null = null;
|
|
588
|
-
let gotAnOp = false;
|
|
589
|
-
let numberOfOpsToClear = 0;
|
|
590
|
-
for await (const rawOp of query.stream()) {
|
|
591
|
-
const op = col.fromPartialPersistedDocument(rawOp);
|
|
592
|
-
|
|
593
|
-
if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
|
|
594
|
-
checksum = utils.addChecksums(checksum, Number(op.checksum));
|
|
595
|
-
lastOp = op;
|
|
596
|
-
numberOfOpsToClear += 1;
|
|
597
|
-
if (op.op != 'CLEAR') {
|
|
598
|
-
gotAnOp = true;
|
|
599
|
-
}
|
|
600
|
-
if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) {
|
|
601
|
-
targetOp = op.target_op;
|
|
602
|
-
}
|
|
603
|
-
} else {
|
|
604
|
-
throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`);
|
|
605
|
-
}
|
|
606
|
-
}
|
|
607
|
-
if (!gotAnOp) {
|
|
608
|
-
done = true;
|
|
609
|
-
return;
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
|
|
613
|
-
await col.collection.deleteMany(
|
|
614
|
-
{
|
|
615
|
-
_id: {
|
|
616
|
-
$gte: col.minId,
|
|
617
|
-
$lte: col.docId(lastOp!.o)
|
|
618
|
-
}
|
|
619
|
-
},
|
|
620
|
-
{ session }
|
|
621
|
-
);
|
|
622
|
-
|
|
623
|
-
const op = col.toPersistedDocument({
|
|
624
|
-
o: lastOp!.o,
|
|
625
|
-
op: 'CLEAR',
|
|
626
|
-
checksum: BigInt(checksum),
|
|
627
|
-
data: null,
|
|
628
|
-
target_op: targetOp
|
|
629
|
-
});
|
|
630
|
-
await col.collection.insertOne(op, { session });
|
|
631
|
-
|
|
632
|
-
opCountDiff = -numberOfOpsToClear + 1;
|
|
633
|
-
},
|
|
634
|
-
{
|
|
635
|
-
writeConcern: { w: 'majority' },
|
|
636
|
-
readConcern: { level: 'snapshot' }
|
|
637
|
-
}
|
|
638
|
-
);
|
|
639
|
-
// Update outside the transaction, since the transaction can be retried multiple times.
|
|
640
|
-
currentState.opCount += opCountDiff;
|
|
641
|
-
}
|
|
642
|
-
} finally {
|
|
643
|
-
await session.endSession();
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
|
|
647
425
|
protected async updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]) {
|
|
648
426
|
const checksums = await this.computeChecksumsForBuckets(buckets);
|
|
649
427
|
const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
|
|
@@ -684,23 +462,31 @@ export abstract class MongoCompactor {
|
|
|
684
462
|
await this.flushBucketStateUpdates();
|
|
685
463
|
}
|
|
686
464
|
|
|
687
|
-
protected formatBucketDataKey(doc: Pick<BucketDataDoc, 'bucketKey' | 'o'>) {
|
|
688
|
-
return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`;
|
|
689
|
-
}
|
|
690
|
-
|
|
691
465
|
protected abstract writeBucketStateUpdates(): Promise<void>;
|
|
692
466
|
protected abstract computeChecksumsForBuckets(
|
|
693
467
|
buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]
|
|
694
468
|
): Promise<storage.PartialChecksumMap>;
|
|
695
469
|
protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document;
|
|
696
|
-
|
|
697
|
-
protected abstract getBucketDataContext(
|
|
698
|
-
bucket: string,
|
|
699
|
-
definitionId: BucketDefinitionId | null
|
|
700
|
-
): Promise<SingleBucketStore | null>;
|
|
701
470
|
}
|
|
702
471
|
|
|
703
472
|
export interface BucketDataCollectionContext<TBucketData extends mongo.Document> {
|
|
704
473
|
bucketKey: BucketKey;
|
|
705
474
|
collection: mongo.Collection<TBucketData>;
|
|
706
475
|
}
|
|
476
|
+
|
|
477
|
+
function compactionRetryReason(error: unknown): string | null {
|
|
478
|
+
if (error instanceof ConcurrentCompactionError) {
|
|
479
|
+
return 'concurrent compaction';
|
|
480
|
+
}
|
|
481
|
+
if (isRetryableObjectStorageError(error)) {
|
|
482
|
+
return 'transient object storage failure';
|
|
483
|
+
}
|
|
484
|
+
if (isMongoServerError(error)) {
|
|
485
|
+
return 'MongoDB failure';
|
|
486
|
+
}
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
function errorMessage(error: unknown): string {
|
|
491
|
+
return error instanceof Error ? error.message : String(error);
|
|
492
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { SyncConfigWithRequiredMapping } from '@powersync/service-core';
|
|
3
|
+
import {
|
|
4
|
+
BucketDataScope,
|
|
5
|
+
BucketDataSource,
|
|
6
|
+
HydrationState,
|
|
7
|
+
ParameterIndexLookupCreator,
|
|
8
|
+
ParameterLookupScope
|
|
9
|
+
} from '@powersync/service-sync-rules';
|
|
10
|
+
|
|
11
|
+
export class MongoHydrationState implements HydrationState {
|
|
12
|
+
private bucketDataSourceSyncConfig = new WeakMap<BucketDataSource, SyncConfigWithRequiredMapping>();
|
|
13
|
+
private parameterIndexLookupSyncConfig = new WeakMap<ParameterIndexLookupCreator, SyncConfigWithRequiredMapping>();
|
|
14
|
+
|
|
15
|
+
constructor(
|
|
16
|
+
readonly syncConfigs: SyncConfigWithRequiredMapping[],
|
|
17
|
+
private readonly replicationStreamId: number
|
|
18
|
+
) {
|
|
19
|
+
for (let syncConfig of syncConfigs) {
|
|
20
|
+
for (let source of syncConfig.syncConfig.config.bucketDataSources) {
|
|
21
|
+
this.bucketDataSourceSyncConfig.set(source, syncConfig);
|
|
22
|
+
}
|
|
23
|
+
for (let source of syncConfig.syncConfig.config.bucketParameterLookupSources) {
|
|
24
|
+
this.parameterIndexLookupSyncConfig.set(source, syncConfig);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
getBucketSourceScope(source: BucketDataSource): BucketDataScope {
|
|
30
|
+
const syncConfig = this.bucketDataSourceSyncConfig.get(source);
|
|
31
|
+
if (syncConfig == null) {
|
|
32
|
+
throw new ServiceAssertionError(`No sync config found for bucket data source ${source.uniqueName}`);
|
|
33
|
+
}
|
|
34
|
+
const mapping = syncConfig.mapping;
|
|
35
|
+
const defId = mapping.bucketSourceId(source);
|
|
36
|
+
// Currently uniqueName is constant for a BucketDataSource within a replication stream. This may change in the
|
|
37
|
+
// future, in which case we need to reconsider the bucketPrefix here.
|
|
38
|
+
// However, the definition may change without changing the name, so we include the defId in the bucketPrefix.
|
|
39
|
+
// defId is not unique across replication streams, so we include the replicationStreamId as well.
|
|
40
|
+
return {
|
|
41
|
+
bucketPrefix: `${source.uniqueName}.${this.replicationStreamId.toString(16)}.${defId}`,
|
|
42
|
+
source
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope {
|
|
47
|
+
const syncConfig = this.parameterIndexLookupSyncConfig.get(source);
|
|
48
|
+
if (syncConfig == null) {
|
|
49
|
+
throw new ServiceAssertionError(
|
|
50
|
+
`No sync config found for parameter index lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId}`
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
const mapping = syncConfig.mapping;
|
|
54
|
+
const defId = mapping.parameterLookupId(source);
|
|
55
|
+
return {
|
|
56
|
+
lookupName: defId,
|
|
57
|
+
queryId: '',
|
|
58
|
+
source
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
}
|