@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
PopulateChecksumCacheOptions,
|
|
18
18
|
PopulateChecksumCacheResults,
|
|
19
19
|
ReplicationCheckpoint,
|
|
20
|
+
ReplicationStreamStorageIds,
|
|
20
21
|
storage,
|
|
21
22
|
utils,
|
|
22
23
|
WatchWriteCheckpointOptions
|
|
@@ -25,21 +26,28 @@ import { HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '
|
|
|
25
26
|
import * as bson from 'bson';
|
|
26
27
|
import { LRUCache } from 'lru-cache';
|
|
27
28
|
import * as timers from 'timers/promises';
|
|
28
|
-
import {
|
|
29
|
+
import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../../types/types.js';
|
|
29
30
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
30
|
-
import {
|
|
31
|
+
import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
|
|
31
32
|
import type { VersionedPowerSyncMongo } from './db.js';
|
|
32
33
|
import { StorageConfig } from './models.js';
|
|
33
34
|
import { MongoBucketBatchOptions } from './MongoBucketBatch.js';
|
|
34
35
|
import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
|
|
35
36
|
import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
|
|
36
37
|
import { MongoParameterCompactor } from './MongoParameterCompactor.js';
|
|
37
|
-
import {
|
|
38
|
+
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
39
|
+
import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
|
|
38
40
|
import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
41
|
+
import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
|
|
39
42
|
|
|
40
43
|
export interface MongoSyncBucketStorageOptions {
|
|
41
|
-
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'
|
|
44
|
+
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
|
|
45
|
+
readPreference?: mongo.ReadPreference;
|
|
46
|
+
checksumCacheTtlMs?: number;
|
|
47
|
+
clearBatchThrottleRate?: number;
|
|
42
48
|
storageConfig: StorageConfig;
|
|
49
|
+
objectStorage?: ObjectStorage;
|
|
50
|
+
inlineThresholdBytes?: number;
|
|
43
51
|
}
|
|
44
52
|
|
|
45
53
|
interface InternalCheckpointChanges extends CheckpointChanges {
|
|
@@ -47,13 +55,6 @@ interface InternalCheckpointChanges extends CheckpointChanges {
|
|
|
47
55
|
invalidateWriteCheckpoints: boolean;
|
|
48
56
|
}
|
|
49
57
|
|
|
50
|
-
interface WriterSyncState {
|
|
51
|
-
lastCheckpointLsn: string | null;
|
|
52
|
-
resumeFromLsn: string | null;
|
|
53
|
-
keepaliveOp: InternalOpId | null;
|
|
54
|
-
syncConfigId?: bson.ObjectId | null;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
58
|
/**
|
|
58
59
|
* Only keep checkpoints around for a minute, before fetching a fresh one.
|
|
59
60
|
*
|
|
@@ -70,34 +71,53 @@ export abstract class MongoSyncBucketStorage
|
|
|
70
71
|
implements storage.SyncRulesBucketStorage
|
|
71
72
|
{
|
|
72
73
|
readonly db: VersionedPowerSyncMongo;
|
|
74
|
+
|
|
73
75
|
[DO_NOT_LOG] = true;
|
|
74
76
|
|
|
75
77
|
readonly checksums: MongoChecksums;
|
|
76
78
|
|
|
77
|
-
|
|
79
|
+
readonly objectStorage?: ObjectStorage;
|
|
80
|
+
readonly inlineThresholdBytes: number;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Canonical parsed sync config sets, keyed by defaultSchema.
|
|
84
|
+
*
|
|
85
|
+
* Entries are never evicted: each parse options value maps to exactly one parsed set for
|
|
86
|
+
* the lifetime of this storage instance, so parsed source objects and mappings always
|
|
87
|
+
* stay associated.
|
|
88
|
+
*/
|
|
89
|
+
private readonly parsedSyncConfigSets = new Map<string, MongoParsedSyncConfigSet>();
|
|
78
90
|
private writeCheckpointAPI: MongoWriteCheckpointAPI;
|
|
79
91
|
public readonly logger: Logger;
|
|
80
92
|
public readonly storageConfig: StorageConfig;
|
|
93
|
+
public readonly readPreference: mongo.ReadPreference | undefined;
|
|
94
|
+
public readonly clearBatchThrottleRate: number;
|
|
81
95
|
#storageInitialized = false;
|
|
82
96
|
|
|
83
97
|
constructor(
|
|
84
98
|
public readonly factory: MongoBucketStorage,
|
|
85
|
-
public readonly
|
|
86
|
-
|
|
87
|
-
public readonly
|
|
99
|
+
public readonly replicationStreamId: number,
|
|
100
|
+
public readonly replicationStream: MongoPersistedReplicationStream,
|
|
101
|
+
public readonly replicationStreamName: string,
|
|
88
102
|
writeCheckpointMode: storage.WriteCheckpointMode | undefined,
|
|
89
103
|
options: MongoSyncBucketStorageOptions
|
|
90
104
|
) {
|
|
91
105
|
super();
|
|
92
106
|
this.storageConfig = options.storageConfig;
|
|
107
|
+
this.objectStorage = options.objectStorage;
|
|
108
|
+
// Keep small chunks inline in MongoDB rather than offloading them to S3.
|
|
109
|
+
// Configurable via object_storage.inline_threshold_bytes.
|
|
110
|
+
this.inlineThresholdBytes = options.inlineThresholdBytes ?? DEFAULT_INLINE_THRESHOLD_BYTES;
|
|
111
|
+
this.readPreference = options.readPreference;
|
|
112
|
+
this.clearBatchThrottleRate = options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
|
|
93
113
|
this.db = factory.db.versioned(this.storageConfig);
|
|
94
114
|
this.checksums = this.createMongoChecksums(options);
|
|
95
115
|
this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
|
|
96
116
|
db: this.db,
|
|
97
117
|
mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
|
|
98
|
-
sync_rules_id:
|
|
118
|
+
sync_rules_id: replicationStreamId
|
|
99
119
|
});
|
|
100
|
-
this.logger =
|
|
120
|
+
this.logger = replicationStream.logger;
|
|
101
121
|
}
|
|
102
122
|
|
|
103
123
|
/**
|
|
@@ -117,40 +137,41 @@ export abstract class MongoSyncBucketStorage
|
|
|
117
137
|
return this.writeCheckpointAPI.writeCheckpointMode;
|
|
118
138
|
}
|
|
119
139
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
return {
|
|
126
|
-
db: this.db,
|
|
127
|
-
group_id: this.group_id,
|
|
128
|
-
mapping: this.mapping
|
|
129
|
-
};
|
|
140
|
+
/**
|
|
141
|
+
* Persisted storage ids of all sync configs in this replication stream. Parse-free.
|
|
142
|
+
*/
|
|
143
|
+
get storageIds(): ReplicationStreamStorageIds {
|
|
144
|
+
return this.replicationStream.storageIds;
|
|
130
145
|
}
|
|
131
146
|
|
|
132
147
|
setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
|
|
133
148
|
this.writeCheckpointAPI.setWriteCheckpointMode(mode);
|
|
134
149
|
}
|
|
135
150
|
|
|
136
|
-
|
|
137
|
-
|
|
151
|
+
createManagedWriteCheckpoints(
|
|
152
|
+
checkpoints: storage.ManagedWriteCheckpointOptions[]
|
|
153
|
+
): Promise<storage.CreateManagedWriteCheckpointsResult> {
|
|
154
|
+
return this.writeCheckpointAPI.createManagedWriteCheckpoints(checkpoints);
|
|
138
155
|
}
|
|
139
156
|
|
|
140
157
|
lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
|
|
141
158
|
return this.writeCheckpointAPI.lastWriteCheckpoint({
|
|
142
159
|
...filters,
|
|
143
|
-
sync_rules_id: this.
|
|
160
|
+
sync_rules_id: this.replicationStreamId
|
|
144
161
|
});
|
|
145
162
|
}
|
|
146
163
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
if (
|
|
150
|
-
|
|
164
|
+
getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
|
|
165
|
+
let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
|
|
166
|
+
if (parsed == null) {
|
|
167
|
+
parsed = this.replicationStream.parsed(options);
|
|
168
|
+
this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
|
|
151
169
|
}
|
|
170
|
+
return parsed;
|
|
171
|
+
}
|
|
152
172
|
|
|
153
|
-
|
|
173
|
+
getParsedSyncRules(options: storage.ParseSyncConfigOptions): HydratedSyncConfig {
|
|
174
|
+
return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
|
|
154
175
|
}
|
|
155
176
|
|
|
156
177
|
async getCheckpoint(): Promise<storage.ReplicationCheckpoint> {
|
|
@@ -169,10 +190,14 @@ export abstract class MongoSyncBucketStorage
|
|
|
169
190
|
}
|
|
170
191
|
|
|
171
192
|
const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined;
|
|
193
|
+
const clusterTime = session.clusterTime;
|
|
172
194
|
if (snapshotTime == null) {
|
|
173
195
|
throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
|
|
174
196
|
}
|
|
175
|
-
|
|
197
|
+
if (clusterTime == null) {
|
|
198
|
+
throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
|
|
199
|
+
}
|
|
200
|
+
return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime);
|
|
176
201
|
});
|
|
177
202
|
}
|
|
178
203
|
|
|
@@ -183,37 +208,42 @@ export abstract class MongoSyncBucketStorage
|
|
|
183
208
|
return;
|
|
184
209
|
}
|
|
185
210
|
|
|
186
|
-
await this.db.initializeStreamStorage(this.
|
|
211
|
+
await this.db.initializeStreamStorage(this.replicationStreamId);
|
|
187
212
|
await this.initializeVersionStorage();
|
|
188
213
|
this.#storageInitialized = true;
|
|
189
214
|
}
|
|
190
215
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
const state = await this.getWriterSyncState();
|
|
216
|
+
/**
|
|
217
|
+
* Create the version-specific writer. Implementations fetch their own resume state
|
|
218
|
+
* (e.g. resume LSN, v1 keepalive op) and construct the batch from
|
|
219
|
+
* {@link writerBatchOptions} plus the version-specific fields.
|
|
220
|
+
*/
|
|
221
|
+
protected abstract createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
|
|
198
222
|
|
|
199
|
-
|
|
223
|
+
/**
|
|
224
|
+
* The version-independent part of the batch options.
|
|
225
|
+
*/
|
|
226
|
+
protected writerBatchOptions(options: storage.CreateWriterOptions): Omit<MongoBucketBatchOptions, 'resumeFromLsn'> {
|
|
227
|
+
return {
|
|
200
228
|
logger: options.logger ?? this.logger,
|
|
201
229
|
db: this.db,
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
slotName: this.slot_name,
|
|
206
|
-
lastCheckpointLsn: state.lastCheckpointLsn,
|
|
207
|
-
resumeFromLsn: state.resumeFromLsn,
|
|
208
|
-
keepaliveOp: state.keepaliveOp,
|
|
230
|
+
parsedSyncConfig: this.getParsedSyncConfigSet(options),
|
|
231
|
+
replicationStreamId: this.replicationStreamId,
|
|
232
|
+
replicationStreamName: this.replicationStreamName,
|
|
209
233
|
storeCurrentData: options.storeCurrentData,
|
|
210
234
|
skipExistingRows: options.skipExistingRows ?? false,
|
|
211
235
|
markRecordUnavailable: options.markRecordUnavailable,
|
|
212
236
|
hooks: options.hooks,
|
|
213
|
-
|
|
214
|
-
|
|
237
|
+
tracer: options.tracer,
|
|
238
|
+
objectStorage: this.objectStorage,
|
|
239
|
+
inlineThresholdBytes: this.inlineThresholdBytes
|
|
215
240
|
};
|
|
216
|
-
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
|
|
244
|
+
await this.initializeStorage();
|
|
245
|
+
|
|
246
|
+
const writer = await this.createWriterImpl(options);
|
|
217
247
|
this.iterateListeners((cb) => cb.batchStarted?.(writer));
|
|
218
248
|
return writer;
|
|
219
249
|
}
|
|
@@ -243,24 +273,31 @@ export abstract class MongoSyncBucketStorage
|
|
|
243
273
|
}
|
|
244
274
|
|
|
245
275
|
protected abstract getBucketDataBatchImpl(
|
|
246
|
-
checkpoint:
|
|
276
|
+
checkpoint: MongoReplicationCheckpoint,
|
|
247
277
|
dataBuckets: storage.BucketDataRequest[],
|
|
248
278
|
options?: storage.BucketDataBatchOptions
|
|
249
|
-
): AsyncIterable<storage.SyncBucketDataChunk>;
|
|
279
|
+
): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd>;
|
|
250
280
|
|
|
251
281
|
async *getBucketDataBatch(
|
|
252
|
-
checkpoint:
|
|
282
|
+
checkpoint: storage.ReplicationCheckpoint,
|
|
253
283
|
dataBuckets: storage.BucketDataRequest[],
|
|
254
284
|
options?: storage.BucketDataBatchOptions
|
|
255
|
-
): AsyncIterable<storage.SyncBucketDataChunk> {
|
|
256
|
-
yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options);
|
|
285
|
+
): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
|
|
286
|
+
yield* this.getBucketDataBatchImpl(checkpoint as MongoReplicationCheckpoint, dataBuckets, options);
|
|
257
287
|
}
|
|
258
288
|
|
|
259
289
|
async getChecksums(
|
|
260
|
-
checkpoint:
|
|
261
|
-
buckets: storage.BucketChecksumRequest[]
|
|
290
|
+
checkpoint: storage.ReplicationCheckpoint,
|
|
291
|
+
buckets: storage.BucketChecksumRequest[],
|
|
292
|
+
options?: storage.BucketChecksumOptions
|
|
262
293
|
): Promise<utils.ChecksumMap> {
|
|
263
|
-
|
|
294
|
+
const mongoCheckpoint = checkpoint as MongoReplicationCheckpoint;
|
|
295
|
+
const snapshotTime = mongoCheckpoint.snapshotTime; // May be undefined in tests
|
|
296
|
+
return this.checksums.getChecksums(checkpoint.checkpoint, buckets, {
|
|
297
|
+
snapshotTime,
|
|
298
|
+
clusterTime: mongoCheckpoint.clusterTime,
|
|
299
|
+
readPreference: options?.requestHint == 'bulk' ? this.readPreference : undefined
|
|
300
|
+
});
|
|
264
301
|
}
|
|
265
302
|
|
|
266
303
|
clearChecksumCache() {
|
|
@@ -277,9 +314,9 @@ export abstract class MongoSyncBucketStorage
|
|
|
277
314
|
await this.db.notifyCheckpoint();
|
|
278
315
|
}
|
|
279
316
|
|
|
280
|
-
protected abstract getStatusImpl(): Promise<storage.
|
|
317
|
+
protected abstract getStatusImpl(): Promise<storage.ReplicationStreamStatus>;
|
|
281
318
|
|
|
282
|
-
async getStatus(): Promise<storage.
|
|
319
|
+
async getStatus(): Promise<storage.ReplicationStreamStatus> {
|
|
283
320
|
return this.getStatusImpl();
|
|
284
321
|
}
|
|
285
322
|
|
|
@@ -312,28 +349,11 @@ export abstract class MongoSyncBucketStorage
|
|
|
312
349
|
this.#storageInitialized = false;
|
|
313
350
|
}
|
|
314
351
|
|
|
315
|
-
protected async clearDeleteMany(
|
|
316
|
-
label: string,
|
|
317
|
-
operation: () => Promise<mongo.DeleteResult>,
|
|
318
|
-
signal?: AbortSignal
|
|
319
|
-
): Promise<void> {
|
|
320
|
-
await retryOnMongoMaxTimeMSExpired(operation, {
|
|
321
|
-
signal,
|
|
322
|
-
abortMessage: 'Aborted clearing data',
|
|
323
|
-
retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5,
|
|
324
|
-
onRetry: () => {
|
|
325
|
-
this.logger.info(
|
|
326
|
-
`Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`
|
|
327
|
-
);
|
|
328
|
-
}
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
|
|
332
352
|
async reportError(e: any): Promise<void> {
|
|
333
353
|
const message = String(e.message ?? 'Replication failure');
|
|
334
354
|
await this.db.sync_rules.updateOne(
|
|
335
355
|
{
|
|
336
|
-
_id: this.
|
|
356
|
+
_id: this.replicationStreamId
|
|
337
357
|
},
|
|
338
358
|
{
|
|
339
359
|
$set: {
|
|
@@ -357,11 +377,21 @@ export abstract class MongoSyncBucketStorage
|
|
|
357
377
|
}
|
|
358
378
|
}
|
|
359
379
|
|
|
380
|
+
/**
|
|
381
|
+
* The highest op id persisted for this stream, whether or not covered by a checkpoint.
|
|
382
|
+
*
|
|
383
|
+
* Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after
|
|
384
|
+
* initial replication but before the first checkpoint exists.
|
|
385
|
+
*/
|
|
386
|
+
protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
|
|
387
|
+
|
|
360
388
|
async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
|
|
361
389
|
this.logger.info(`Populating persistent checksum cache...`);
|
|
362
390
|
const start = Date.now();
|
|
391
|
+
const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
|
|
363
392
|
const compactor = this.createMongoCompactor({
|
|
364
393
|
...options,
|
|
394
|
+
maxOpId,
|
|
365
395
|
memoryLimitMB: 0,
|
|
366
396
|
logger: this.logger
|
|
367
397
|
});
|
|
@@ -413,7 +443,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
413
443
|
for await (const nextCheckpoint of iter) {
|
|
414
444
|
if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) {
|
|
415
445
|
writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
|
|
416
|
-
sync_rules_id: this.
|
|
446
|
+
sync_rules_id: this.replicationStreamId,
|
|
417
447
|
user_id: options.user_id,
|
|
418
448
|
heads: {
|
|
419
449
|
'1': nextCheckpoint.lsn
|
|
@@ -446,7 +476,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
446
476
|
let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null;
|
|
447
477
|
if (updates.invalidateWriteCheckpoints) {
|
|
448
478
|
updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
|
|
449
|
-
sync_rules_id: this.
|
|
479
|
+
sync_rules_id: this.replicationStreamId,
|
|
450
480
|
user_id: options.user_id,
|
|
451
481
|
heads: {
|
|
452
482
|
'1': nextCheckpoint.lsn!
|
|
@@ -566,6 +596,9 @@ export abstract class MongoSyncBucketStorage
|
|
|
566
596
|
>({
|
|
567
597
|
max: 50,
|
|
568
598
|
maxSize: 12 * 1024 * 1024,
|
|
599
|
+
// When we have more fetches than the cache size, complete the fetches instead
|
|
600
|
+
// of failing with Error('evicted').
|
|
601
|
+
ignoreFetchAbort: true,
|
|
569
602
|
sizeCalculation: (value: InternalCheckpointChanges) => {
|
|
570
603
|
const paramSize = [...value.updatedParameterLookups].reduce<number>((a, b) => a + b.length, 0);
|
|
571
604
|
const bucketSize = [...value.updatedDataBuckets].reduce<number>((a, b) => a + b.length, 0);
|
|
@@ -603,7 +636,8 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
603
636
|
storage: MongoSyncBucketStorage,
|
|
604
637
|
public readonly checkpoint: InternalOpId,
|
|
605
638
|
public readonly lsn: string | null,
|
|
606
|
-
public snapshotTime: mongo.Timestamp
|
|
639
|
+
public snapshotTime: mongo.Timestamp,
|
|
640
|
+
public clusterTime: mongo.ClusterTime
|
|
607
641
|
) {
|
|
608
642
|
this.#storage = storage;
|
|
609
643
|
}
|
|
@@ -617,7 +651,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
617
651
|
readonly checkpoint: InternalOpId = 0n;
|
|
618
652
|
readonly lsn: string | null = null;
|
|
619
653
|
|
|
620
|
-
async getParameterSets(_lookups: ScopedParameterLookup[]): Promise<ParameterLookupRows[]> {
|
|
654
|
+
async getParameterSets(_lookups: ScopedParameterLookup[], _limit: number): Promise<ParameterLookupRows[]> {
|
|
621
655
|
return [];
|
|
622
656
|
}
|
|
623
657
|
}
|
|
@@ -17,12 +17,12 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
17
17
|
*/
|
|
18
18
|
static async createLock(
|
|
19
19
|
db: VersionedPowerSyncMongo,
|
|
20
|
-
sync_rules: storage.
|
|
20
|
+
sync_rules: storage.PersistedReplicationStream,
|
|
21
21
|
session?: mongo.ClientSession
|
|
22
22
|
): Promise<MongoSyncRulesLock> {
|
|
23
23
|
const lockId = crypto.randomBytes(8).toString('hex');
|
|
24
24
|
const doc = await db.sync_rules.findOneAndUpdate(
|
|
25
|
-
{ _id: sync_rules.
|
|
25
|
+
{ _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
|
|
26
26
|
{
|
|
27
27
|
$set: {
|
|
28
28
|
lock: {
|
|
@@ -40,7 +40,10 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
40
40
|
|
|
41
41
|
if (doc == null) {
|
|
42
42
|
// Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
|
|
43
|
-
const heldLock = await db.sync_rules.findOne(
|
|
43
|
+
const heldLock = await db.sync_rules.findOne(
|
|
44
|
+
{ _id: sync_rules.replicationStreamId },
|
|
45
|
+
{ projection: { lock: 1 }, session }
|
|
46
|
+
);
|
|
44
47
|
if (heldLock?.lock?.expires_at) {
|
|
45
48
|
throw new ServiceError(
|
|
46
49
|
ErrorCode.PSYNC_S1003,
|
|
@@ -51,7 +54,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
51
54
|
}
|
|
52
55
|
}
|
|
53
56
|
sync_rules.logger.info(`Locked replication stream for processing`);
|
|
54
|
-
return new MongoSyncRulesLock(db, sync_rules.
|
|
57
|
+
return new MongoSyncRulesLock(db, sync_rules.replicationStreamId, lockId, sync_rules.logger);
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
constructor(
|