@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { BucketDefinitionMapping, storage, SyncConfigWithMapping } from '@powersync/service-core';
|
|
2
|
+
import { EventDefinitionId, HydratedEventDescriptor, HydratedSyncConfig, HydrationState, SyncConfigWithErrors } from '@powersync/service-sync-rules';
|
|
3
|
+
import { StorageConfig } from './models.js';
|
|
4
|
+
export declare class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet {
|
|
5
|
+
#private;
|
|
6
|
+
readonly replicationStreamId: number;
|
|
7
|
+
readonly hydrationState: HydrationState;
|
|
8
|
+
readonly syncConfigs: SyncConfigWithErrors[];
|
|
9
|
+
readonly replicationStreamName: string;
|
|
10
|
+
readonly mapping: BucketDefinitionMapping;
|
|
11
|
+
constructor(replicationStreamId: number, storageConfig: StorageConfig, slotName: string, syncConfigs: SyncConfigWithMapping[]);
|
|
12
|
+
get hydratedSyncConfig(): HydratedSyncConfig;
|
|
13
|
+
/**
|
|
14
|
+
* Hydrated events for the replication stream, keyed by their assigned storage id.
|
|
15
|
+
*
|
|
16
|
+
* Each config's events are resolved against that config's own (single-config) mapping, so the resolution is
|
|
17
|
+
* unambiguous, and the result is deduplicated by assigned id: unchanged events shared across configs collapse to
|
|
18
|
+
* one entry, while a changed event keeps a separate entry under its new id.
|
|
19
|
+
*/
|
|
20
|
+
get eventById(): ReadonlyMap<EventDefinitionId, HydratedEventDescriptor>;
|
|
21
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import * as sqlite from 'node:sqlite';
|
|
2
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { MultiSyncConfigBucketDefinitionMapping, SingleSyncConfigBucketDefinitionMapping } from '@powersync/service-core';
|
|
4
|
+
import { CompatibilityOption, DEFAULT_HYDRATION_STATE, HydratedSyncConfig, nodeSqlite, versionedHydrationState } from '@powersync/service-sync-rules';
|
|
5
|
+
import { MongoHydrationState } from './MongoHydrationState.js';
|
|
6
|
+
export class MongoParsedSyncConfigSet {
|
|
7
|
+
replicationStreamId;
|
|
8
|
+
hydrationState;
|
|
9
|
+
syncConfigs;
|
|
10
|
+
replicationStreamName;
|
|
11
|
+
mapping;
|
|
12
|
+
#configsWithMapping;
|
|
13
|
+
constructor(replicationStreamId, storageConfig, slotName, syncConfigs) {
|
|
14
|
+
this.replicationStreamId = replicationStreamId;
|
|
15
|
+
this.replicationStreamName = slotName;
|
|
16
|
+
this.#configsWithMapping = [...syncConfigs];
|
|
17
|
+
this.syncConfigs = syncConfigs.map((config) => config.syncConfig);
|
|
18
|
+
if (this.syncConfigs.length == 0) {
|
|
19
|
+
throw new ServiceAssertionError(`At least one sync config is required`);
|
|
20
|
+
}
|
|
21
|
+
const [firstConfig] = this.syncConfigs;
|
|
22
|
+
const compatibility = firstConfig.config.compatibility;
|
|
23
|
+
for (const config of this.syncConfigs) {
|
|
24
|
+
if (config.config.compatibility.equals(compatibility)) {
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
throw new ServiceAssertionError(`All sync configs in a replication stream must use the same compatibility options`);
|
|
28
|
+
}
|
|
29
|
+
if (storageConfig.incrementalReprocessing) {
|
|
30
|
+
if (syncConfigs.some((c) => c.mapping == null)) {
|
|
31
|
+
throw new ServiceAssertionError(`mapping is required for v3 storage`);
|
|
32
|
+
}
|
|
33
|
+
const mappedConfigs = syncConfigs;
|
|
34
|
+
this.hydrationState = new MongoHydrationState(mappedConfigs, this.replicationStreamId);
|
|
35
|
+
this.mapping = new MultiSyncConfigBucketDefinitionMapping(mappedConfigs);
|
|
36
|
+
}
|
|
37
|
+
else if (!compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !storageConfig.versionedBuckets) {
|
|
38
|
+
const [syncConfig] = syncConfigs;
|
|
39
|
+
if (syncConfigs.length != 1 || syncConfig == null) {
|
|
40
|
+
throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
|
|
41
|
+
}
|
|
42
|
+
this.hydrationState = DEFAULT_HYDRATION_STATE;
|
|
43
|
+
this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
const [syncConfig] = syncConfigs;
|
|
47
|
+
if (syncConfigs.length != 1 || syncConfig == null) {
|
|
48
|
+
throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
|
|
49
|
+
}
|
|
50
|
+
this.hydrationState = versionedHydrationState(this.replicationStreamId);
|
|
51
|
+
this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
#hydratedSyncConfig;
|
|
55
|
+
get hydratedSyncConfig() {
|
|
56
|
+
this.#hydratedSyncConfig ??= new HydratedSyncConfig({
|
|
57
|
+
definitions: this.syncConfigs.map((config) => config.config),
|
|
58
|
+
createParams: {
|
|
59
|
+
hydrationState: this.hydrationState,
|
|
60
|
+
sqlite: nodeSqlite(sqlite)
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
return this.#hydratedSyncConfig;
|
|
64
|
+
}
|
|
65
|
+
#eventById;
|
|
66
|
+
/**
|
|
67
|
+
* Hydrated events for the replication stream, keyed by their assigned storage id.
|
|
68
|
+
*
|
|
69
|
+
* Each config's events are resolved against that config's own (single-config) mapping, so the resolution is
|
|
70
|
+
* unambiguous, and the result is deduplicated by assigned id: unchanged events shared across configs collapse to
|
|
71
|
+
* one entry, while a changed event keeps a separate entry under its new id.
|
|
72
|
+
*/
|
|
73
|
+
get eventById() {
|
|
74
|
+
if (this.#eventById == null) {
|
|
75
|
+
const map = new Map();
|
|
76
|
+
const byDefinition = this.hydratedSyncConfig.eventDescriptorsByDefinition;
|
|
77
|
+
for (const config of this.#configsWithMapping) {
|
|
78
|
+
if (config.mapping == null) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
for (const event of byDefinition.get(config.syncConfig.config) ?? []) {
|
|
82
|
+
const id = config.mapping.eventId(event);
|
|
83
|
+
if (!map.has(id)) {
|
|
84
|
+
map.set(id, event);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
this.#eventById = map;
|
|
89
|
+
}
|
|
90
|
+
return this.#eventById;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=MongoParsedSyncConfigSet.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MongoParsedSyncConfigSet.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoParsedSyncConfigSet.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAEL,sCAAsC,EACtC,uCAAuC,EAIxC,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EAGvB,kBAAkB,EAElB,UAAU,EAEV,uBAAuB,EACxB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAE/D,MAAM,OAAO,wBAAwB;IASjB;IARF,cAAc,CAAiB;IAC/B,WAAW,CAAyB;IACpC,qBAAqB,CAAS;IAC9B,OAAO,CAA0B;IAExC,mBAAmB,CAAmC;IAE/D,YACkB,mBAA2B,EAC3C,aAA4B,EAC5B,QAAgB,EAChB,WAAoC;QAHpB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAK3C,IAAI,CAAC,qBAAqB,GAAG,QAAQ,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAClE,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,qBAAqB,CAAC,sCAAsC,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;QACvC,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC;QACvD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,IAAI,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;gBACtD,SAAS;YACX,CAAC;YACD,MAAM,IAAI,qBAAqB,CAC7B,kFAAkF,CACnF,CAAC;QACJ,CAAC;QAED,IAAI,aAAa,CAAC,uBAAuB,EAAE,CAAC;YAC1C,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;YACxE,CAAC;YACD,MAAM,aAAa,GAAG,WAA8C,CAAC;YACrE,IAAI,CAAC,cAAc,GAAG,IAAI,mBAAmB,CAAC,aAAa,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACvF,IAAI,CAAC,OAAO,GAAG,IAAI,sCAAsC,CAAC,aAAa,CAAC,CAAC;QAC3E,CAAC;aAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAAC;YAC/G,MAAM,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;YACjC,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;gBAClD,MAAM,IAAI,qBAAqB,CAAC,0DAA0D,CAAC,CAAC;YAC9F,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,uBAAuB,CAAC;YAC9C,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,IAAI,uCAAuC,EAAE,CAAC;QACrF,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;YACjC,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;gBAClD,MAAM,IAAI,qBAAqB,CAAC,0DAA0D,CAAC,CAAC;YAC9F,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,uBAAuB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxE,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,IAAI,uCAAuC,EAAE,CAAC;QACrF,CAAC;IACH,CAAC;IAED,mBAAmB,CAAiC;IAEpD,IAAI,kBAAkB;QACpB,IAAI,CAAC,mBAAmB,KAAK,IAAI,kBAAkB,CAAC;YAClD,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;YAC5D,YAAY,EAAE;gBACZ,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC;aAC3B;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,UAAU,CAAsE;IAEhF;;;;;;OAMG;IACH,IAAI,SAAS;QACX,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,GAAG,EAA8C,CAAC;YAClE,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,4BAA4B,CAAC;YAC1E,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC9C,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;oBAC3B,SAAS;gBACX,CAAC;gBACD,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;oBACrE,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACzC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;wBACjB,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;QACxB,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ReplicationStreamStorageIds, storage } from '@powersync/service-core';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
import { ReplicationStreamDocumentV3, SyncConfigDefinition } from '../storage-index.js';
|
|
5
|
+
import { PowerSyncMongo } from './db.js';
|
|
6
|
+
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
7
|
+
import { MongoPersistedSyncConfigContentBase } from './MongoPersistedSyncConfigContent.js';
|
|
8
|
+
import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
|
|
9
|
+
import { SyncRuleDocumentV1 } from './v1/models.js';
|
|
10
|
+
export declare class MongoPersistedReplicationStream extends storage.PersistedReplicationStream {
|
|
11
|
+
#private;
|
|
12
|
+
private readonly db;
|
|
13
|
+
private readonly doc;
|
|
14
|
+
private readonly configs;
|
|
15
|
+
current_lock: MongoSyncRulesLock | null;
|
|
16
|
+
readonly syncConfigContent: readonly MongoPersistedSyncConfigContentBase[];
|
|
17
|
+
constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1 | ReplicationStreamDocumentV3, configs?: SyncConfigDefinition[]);
|
|
18
|
+
getStorageConfig(): import("./models.js").StorageConfig;
|
|
19
|
+
private createSyncConfigContent;
|
|
20
|
+
get syncConfigIds(): bson.ObjectId[];
|
|
21
|
+
/**
|
|
22
|
+
* Persisted storage ids for all sync configs in this replication stream.
|
|
23
|
+
*
|
|
24
|
+
* Derived from the persisted rule_mapping documents - requires no parsed sync configs.
|
|
25
|
+
*/
|
|
26
|
+
get storageIds(): ReplicationStreamStorageIds;
|
|
27
|
+
/**
|
|
28
|
+
* Parse the sync configs for this replication stream.
|
|
29
|
+
*
|
|
30
|
+
* This creates a fresh parse on every call. Do not call this from operational paths -
|
|
31
|
+
* use {@link MongoSyncBucketStorage.getParsedSyncConfigSet} instead, which returns a
|
|
32
|
+
* canonical instance so that parsed source objects and mappings stay associated.
|
|
33
|
+
*/
|
|
34
|
+
parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet;
|
|
35
|
+
lock(session?: mongo.ClientSession): Promise<MongoSyncRulesLock>;
|
|
36
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { ReplicationStreamStorageIds, SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
|
|
3
|
+
import { getMongoStorageConfig } from './models.js';
|
|
4
|
+
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
5
|
+
import { MongoPersistedSyncConfigContentV1, MongoPersistedSyncConfigContentV3 } from './MongoPersistedSyncConfigContent.js';
|
|
6
|
+
import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
|
|
7
|
+
export class MongoPersistedReplicationStream extends storage.PersistedReplicationStream {
|
|
8
|
+
db;
|
|
9
|
+
doc;
|
|
10
|
+
configs;
|
|
11
|
+
current_lock = null;
|
|
12
|
+
syncConfigContent;
|
|
13
|
+
constructor(db, doc, configs = []) {
|
|
14
|
+
const storageVersion = doc.storage_version ?? storage.LEGACY_STORAGE_VERSION;
|
|
15
|
+
const replicationJobId = configs.length == 0
|
|
16
|
+
? String(doc._id)
|
|
17
|
+
: `${doc._id}:${configs
|
|
18
|
+
.map((config) => config._id.toHexString())
|
|
19
|
+
.sort()
|
|
20
|
+
.join(',')}`;
|
|
21
|
+
super({
|
|
22
|
+
replicationStreamId: doc._id,
|
|
23
|
+
replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
|
|
24
|
+
state: doc.state,
|
|
25
|
+
storageVersion,
|
|
26
|
+
replicationJobId
|
|
27
|
+
});
|
|
28
|
+
this.db = db;
|
|
29
|
+
this.doc = doc;
|
|
30
|
+
this.configs = configs;
|
|
31
|
+
this.syncConfigContent = this.createSyncConfigContent();
|
|
32
|
+
}
|
|
33
|
+
getStorageConfig() {
|
|
34
|
+
return getMongoStorageConfig(this.storageVersion);
|
|
35
|
+
}
|
|
36
|
+
createSyncConfigContent() {
|
|
37
|
+
if (this.getStorageConfig().incrementalReprocessing) {
|
|
38
|
+
if (this.configs.length == 0) {
|
|
39
|
+
throw new ServiceAssertionError(`Cannot create v3 storage without sync config definitions`);
|
|
40
|
+
}
|
|
41
|
+
return this.configs.map((config) => new MongoPersistedSyncConfigContentV3(this.db, this.doc, config));
|
|
42
|
+
}
|
|
43
|
+
return [new MongoPersistedSyncConfigContentV1(this.db, this.doc)];
|
|
44
|
+
}
|
|
45
|
+
get syncConfigIds() {
|
|
46
|
+
return this.configs.map((config) => config._id);
|
|
47
|
+
}
|
|
48
|
+
#storageIds;
|
|
49
|
+
/**
|
|
50
|
+
* Persisted storage ids for all sync configs in this replication stream.
|
|
51
|
+
*
|
|
52
|
+
* Derived from the persisted rule_mapping documents - requires no parsed sync configs.
|
|
53
|
+
*/
|
|
54
|
+
get storageIds() {
|
|
55
|
+
this.#storageIds ??= new ReplicationStreamStorageIds(this.syncConfigContent.map((content) => content.mapping));
|
|
56
|
+
return this.#storageIds;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Parse the sync configs for this replication stream.
|
|
60
|
+
*
|
|
61
|
+
* This creates a fresh parse on every call. Do not call this from operational paths -
|
|
62
|
+
* use {@link MongoSyncBucketStorage.getParsedSyncConfigSet} instead, which returns a
|
|
63
|
+
* canonical instance so that parsed source objects and mappings stay associated.
|
|
64
|
+
*/
|
|
65
|
+
parsed(options) {
|
|
66
|
+
const storageConfig = this.getStorageConfig();
|
|
67
|
+
if (!storageConfig.incrementalReprocessing) {
|
|
68
|
+
return this.syncConfigContent[0].parsed(options);
|
|
69
|
+
}
|
|
70
|
+
const syncConfigs = this.configs.map((config) => {
|
|
71
|
+
return {
|
|
72
|
+
syncConfigId: config._id.toHexString(),
|
|
73
|
+
syncConfig: storage.parsePersistedSyncConfigContent({
|
|
74
|
+
content: config.content,
|
|
75
|
+
compiledPlan: config.serialized_plan ?? null,
|
|
76
|
+
storageVersion: this.storageVersion,
|
|
77
|
+
parseOptions: options
|
|
78
|
+
}),
|
|
79
|
+
mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping)
|
|
80
|
+
};
|
|
81
|
+
});
|
|
82
|
+
return new MongoParsedSyncConfigSet(this.replicationStreamId, storageConfig, this.replicationStreamName, syncConfigs);
|
|
83
|
+
}
|
|
84
|
+
async lock(session) {
|
|
85
|
+
const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session);
|
|
86
|
+
this.current_lock = lock;
|
|
87
|
+
return lock;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=MongoPersistedReplicationStream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MongoPersistedReplicationStream.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoPersistedReplicationStream.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,2BAA2B,EAAE,uCAAuC,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAIxH,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAEL,iCAAiC,EACjC,iCAAiC,EAClC,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAG7D,MAAM,OAAO,+BAAgC,SAAQ,OAAO,CAAC,0BAA0B;IAKlE;IACA;IACA;IANZ,YAAY,GAA8B,IAAI,CAAC;IACtC,iBAAiB,CAAiD;IAElF,YACmB,EAAkB,EAClB,GAAqD,EACrD,UAAkC,EAAE;QAErD,MAAM,cAAc,GAAG,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,sBAAsB,CAAC;QAC7E,MAAM,gBAAgB,GACpB,OAAO,CAAC,MAAM,IAAI,CAAC;YACjB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YACjB,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,OAAO;iBAClB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;iBACzC,IAAI,EAAE;iBACN,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAErB,KAAK,CAAC;YACJ,mBAAmB,EAAE,GAAG,CAAC,GAAG;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS,IAAI,aAAa,GAAG,CAAC,GAAG,EAAE;YAC9D,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,cAAc;YACd,gBAAgB;SACjB,CAAC,CAAC;QAnBc,OAAE,GAAF,EAAE,CAAgB;QAClB,QAAG,GAAH,GAAG,CAAkD;QACrD,YAAO,GAAP,OAAO,CAA6B;QAmBrD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC;IAC1D,CAAC;IAED,gBAAgB;QACd,OAAO,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC;IAEO,uBAAuB;QAC7B,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC,uBAAuB,EAAE,CAAC;YACpD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,qBAAqB,CAAC,0DAA0D,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CACrB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,iCAAiC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAkC,EAAE,MAAM,CAAC,CAC5G,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,iCAAiC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAyB,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC;IAED,WAAW,CAA0C;IAErD;;;;OAIG;IACH,IAAI,UAAU;QACZ,IAAI,CAAC,WAAW,KAAK,IAAI,2BAA2B,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/G,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,OAAuC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,CAAC,aAAa,CAAC,uBAAuB,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9C,OAAO;gBACL,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE;gBACtC,UAAU,EAAE,OAAO,CAAC,+BAA+B,CAAC;oBAClD,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,YAAY,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;oBAC5C,cAAc,EAAE,IAAI,CAAC,cAAc;oBACnC,YAAY,EAAE,OAAO;iBACtB,CAAC;gBACF,OAAO,EAAE,uCAAuC,CAAC,oBAAoB,CAAC,MAAM,CAAC,YAAY,CAAC;aAC3F,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,wBAAwB,CACjC,IAAI,CAAC,mBAAmB,EACxB,aAAa,EACb,IAAI,CAAC,qBAAqB,EAC1B,WAAW,CACZ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,OAA6B;QACtC,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5G,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
|
|
2
|
+
import * as bson from 'bson';
|
|
3
|
+
import { ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleDocumentV1 } from '../storage-index.js';
|
|
4
|
+
import { PowerSyncMongo } from './db.js';
|
|
5
|
+
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
6
|
+
export declare abstract class MongoPersistedSyncConfigContentBase extends storage.PersistedSyncConfigContent {
|
|
7
|
+
protected readonly db: PowerSyncMongo;
|
|
8
|
+
readonly mapping: SingleSyncConfigBucketDefinitionMapping;
|
|
9
|
+
readonly syncConfigObjectId: bson.ObjectId | null;
|
|
10
|
+
protected constructor(db: PowerSyncMongo, options: Omit<storage.PersistedSyncConfigContentData, 'syncConfigId'> & {
|
|
11
|
+
mapping: SingleSyncConfigBucketDefinitionMapping;
|
|
12
|
+
syncConfigId: bson.ObjectId | null;
|
|
13
|
+
});
|
|
14
|
+
getStorageConfig(): import("./models.js").StorageConfig;
|
|
15
|
+
parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet;
|
|
16
|
+
}
|
|
17
|
+
export declare class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
|
|
18
|
+
constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1);
|
|
19
|
+
getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null>;
|
|
20
|
+
}
|
|
21
|
+
export declare class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
|
|
22
|
+
readonly syncConfigObjectId: bson.ObjectId;
|
|
23
|
+
constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition);
|
|
24
|
+
getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
|
|
3
|
+
import { getMongoStorageConfig } from './models.js';
|
|
4
|
+
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
5
|
+
export class MongoPersistedSyncConfigContentBase extends storage.PersistedSyncConfigContent {
|
|
6
|
+
db;
|
|
7
|
+
mapping;
|
|
8
|
+
syncConfigObjectId;
|
|
9
|
+
constructor(db, options) {
|
|
10
|
+
const { mapping, syncConfigId, ...base } = options;
|
|
11
|
+
super({
|
|
12
|
+
...base,
|
|
13
|
+
syncConfigId: syncConfigId?.toHexString() ?? null
|
|
14
|
+
});
|
|
15
|
+
this.db = db;
|
|
16
|
+
this.mapping = mapping;
|
|
17
|
+
this.syncConfigObjectId = syncConfigId;
|
|
18
|
+
}
|
|
19
|
+
getStorageConfig() {
|
|
20
|
+
return getMongoStorageConfig(this.storageVersion);
|
|
21
|
+
}
|
|
22
|
+
parsed(options) {
|
|
23
|
+
const parsed = super.parsed(options);
|
|
24
|
+
const storageConfig = this.getStorageConfig();
|
|
25
|
+
const [syncConfig] = parsed.syncConfigs;
|
|
26
|
+
if (syncConfig == null) {
|
|
27
|
+
throw new ServiceAssertionError(`Expected one parsed sync config`);
|
|
28
|
+
}
|
|
29
|
+
if (storageConfig.incrementalReprocessing) {
|
|
30
|
+
const syncConfigId = this.syncConfigId;
|
|
31
|
+
if (syncConfigId == null) {
|
|
32
|
+
throw new ServiceAssertionError(`syncConfigId is required for v3 sync config content`);
|
|
33
|
+
}
|
|
34
|
+
return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
|
|
35
|
+
{ syncConfigId, syncConfig, mapping: this.mapping }
|
|
36
|
+
]);
|
|
37
|
+
}
|
|
38
|
+
return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
|
|
39
|
+
{ syncConfig, mapping: null }
|
|
40
|
+
]);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
|
|
44
|
+
constructor(db, doc) {
|
|
45
|
+
super(db, {
|
|
46
|
+
replicationStreamId: doc._id,
|
|
47
|
+
sync_rules_content: doc.content,
|
|
48
|
+
compiled_plan: doc.serialized_plan ?? null,
|
|
49
|
+
// Handle legacy values
|
|
50
|
+
replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
|
|
51
|
+
storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
|
|
52
|
+
mapping: new SingleSyncConfigBucketDefinitionMapping(),
|
|
53
|
+
syncConfigId: null,
|
|
54
|
+
syncConfigState: doc.state,
|
|
55
|
+
version_label: doc.version_label ?? undefined
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
async getSyncConfigStatus() {
|
|
59
|
+
const doc = await this.db.sync_rules.findOne({ _id: this.replicationStreamId });
|
|
60
|
+
if (doc == null) {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
return syncConfigStatusFromV1(doc);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
|
|
67
|
+
constructor(db, doc, config) {
|
|
68
|
+
const state = doc.sync_configs.find((c) => c._id.equals(config._id));
|
|
69
|
+
if (state == null) {
|
|
70
|
+
throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
|
|
71
|
+
}
|
|
72
|
+
super(db, {
|
|
73
|
+
replicationStreamId: doc._id,
|
|
74
|
+
sync_rules_content: config.content,
|
|
75
|
+
compiled_plan: config.serialized_plan ?? null,
|
|
76
|
+
replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
|
|
77
|
+
storageVersion: doc.storage_version,
|
|
78
|
+
mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
|
|
79
|
+
syncConfigId: config._id,
|
|
80
|
+
syncConfigState: state.state,
|
|
81
|
+
version_label: config.version_label
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
async getSyncConfigStatus() {
|
|
85
|
+
const doc = await this.db.sync_rules.findOne({
|
|
86
|
+
_id: this.replicationStreamId,
|
|
87
|
+
'sync_configs._id': this.syncConfigObjectId
|
|
88
|
+
});
|
|
89
|
+
if (doc == null) {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
const state = doc.sync_configs.find((c) => c._id.equals(this.syncConfigObjectId));
|
|
93
|
+
if (state == null) {
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
return syncConfigStatusFromV3(doc, state);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function syncConfigStatusFromV1(doc) {
|
|
100
|
+
return {
|
|
101
|
+
id: String(doc._id),
|
|
102
|
+
replicationStreamId: doc._id,
|
|
103
|
+
state: doc.state,
|
|
104
|
+
snapshot_done: doc.snapshot_done,
|
|
105
|
+
last_checkpoint_lsn: doc.last_checkpoint_lsn,
|
|
106
|
+
last_fatal_error: doc.last_fatal_error,
|
|
107
|
+
last_fatal_error_ts: doc.last_fatal_error_ts,
|
|
108
|
+
last_keepalive_ts: doc.last_keepalive_ts,
|
|
109
|
+
last_checkpoint_ts: doc.last_checkpoint_ts
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
function syncConfigStatusFromV3(doc, state) {
|
|
113
|
+
return {
|
|
114
|
+
id: state._id.toHexString(),
|
|
115
|
+
replicationStreamId: doc._id,
|
|
116
|
+
state: state.state,
|
|
117
|
+
snapshot_done: state.snapshot_done,
|
|
118
|
+
last_checkpoint_lsn: state.last_checkpoint_lsn,
|
|
119
|
+
last_fatal_error: doc.last_fatal_error,
|
|
120
|
+
last_fatal_error_ts: doc.last_fatal_error_ts,
|
|
121
|
+
last_keepalive_ts: doc.last_keepalive_ts,
|
|
122
|
+
last_checkpoint_ts: doc.last_checkpoint_ts
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=MongoPersistedSyncConfigContent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MongoPersistedSyncConfigContent.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoPersistedSyncConfigContent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,uCAAuC,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAS3F,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,OAAgB,mCAAoC,SAAQ,OAAO,CAAC,0BAA0B;IAK7E;IAJL,OAAO,CAA0C;IACjD,kBAAkB,CAAuB;IAEzD,YACqB,EAAkB,EACrC,OAGC;QAED,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACnD,KAAK,CAAC;YACJ,GAAG,IAAI;YACP,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,IAAI;SAClD,CAAC,CAAC;QAVgB,OAAE,GAAF,EAAE,CAAgB;QAWrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,kBAAkB,GAAG,YAAY,CAAC;IACzC,CAAC;IAED,gBAAgB;QACd,OAAO,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,CAAC,OAAuC;QAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACxC,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,qBAAqB,CAAC,iCAAiC,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,aAAa,CAAC,uBAAuB,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;YACvC,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,qBAAqB,CAAC,qDAAqD,CAAC,CAAC;YACzF,CAAC;YAED,OAAO,IAAI,wBAAwB,CAAC,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,MAAM,CAAC,qBAAqB,EAAE;gBAC3G,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;QACL,CAAC;QAED,OAAO,IAAI,wBAAwB,CAAC,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,MAAM,CAAC,qBAAqB,EAAE;YAC3G,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE;SAC9B,CAAC,CAAC;IACL,CAAC;CACF;AACD,MAAM,OAAO,iCAAkC,SAAQ,mCAAmC;IACxF,YAAY,EAAkB,EAAE,GAAuB;QACrD,KAAK,CAAC,EAAE,EAAE;YACR,mBAAmB,EAAE,GAAG,CAAC,GAAG;YAC5B,kBAAkB,EAAE,GAAG,CAAC,OAAO;YAC/B,aAAa,EAAE,GAAG,CAAC,eAAe,IAAI,IAAI;YAC1C,uBAAuB;YACvB,qBAAqB,EAAE,GAAG,CAAC,SAAS,IAAI,aAAa,GAAG,CAAC,GAAG,EAAE;YAC9D,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,sBAAsB;YACrE,OAAO,EAAE,IAAI,uCAAuC,EAAE;YACtD,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,GAAG,CAAC,KAAK;YAC1B,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAqB,EAAE,GAAG,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;QACpG,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,sBAAsB,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;CACF;AACD,MAAM,OAAO,iCAAkC,SAAQ,mCAAmC;IAGxF,YAAY,EAAkB,EAAE,GAAgC,EAAE,MAA4B;QAC5F,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACrE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,qBAAqB,CAAC,2BAA2B,MAAM,CAAC,GAAG,0BAA0B,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5G,CAAC;QACD,KAAK,CAAC,EAAE,EAAE;YACR,mBAAmB,EAAE,GAAG,CAAC,GAAG;YAC5B,kBAAkB,EAAE,MAAM,CAAC,OAAO;YAClC,aAAa,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;YAE7C,qBAAqB,EAAE,GAAG,CAAC,SAAS,IAAI,aAAa,GAAG,CAAC,GAAG,EAAE;YAC9D,cAAc,EAAE,GAAG,CAAC,eAAe;YACnC,OAAO,EAAE,uCAAuC,CAAC,oBAAoB,CAAC,MAAM,CAAC,YAAY,CAAC;YAC1F,YAAY,EAAE,MAAM,CAAC,GAAG;YACxB,eAAe,EAAE,KAAK,CAAC,KAAK;YAC5B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAA8B;YACxE,GAAG,EAAE,IAAI,CAAC,mBAAmB;YAC7B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;SAC5C,CAAC,CAAC;QACH,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAClF,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,sBAAsB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;CACF;AAED,SAAS,sBAAsB,CAAC,GAAuB;IACrD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACnB,mBAAmB,EAAE,GAAG,CAAC,GAAG;QAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,aAAa,EAAE,GAAG,CAAC,aAAa;QAChC,mBAAmB,EAAE,GAAG,CAAC,mBAAmB;QAC5C,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,mBAAmB,EAAE,GAAG,CAAC,mBAAmB;QAC5C,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;QACxC,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;KAC3C,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,GAAgC,EAChC,KAA4B;IAE5B,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE;QAC3B,mBAAmB,EAAE,GAAG,CAAC,GAAG;QAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,mBAAmB,EAAE,GAAG,CAAC,mBAAmB;QAC5C,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;QACxC,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;KAC3C,CAAC;AACJ,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { POWERSYNC_VERSION } from '@powersync/service-core';
|
|
4
|
-
import { MongoStorageConfig } from '../../types/types.js';
|
|
4
|
+
import { MongoStorageConfig, normalizeClearBatchThrottleRate } from '../../types/types.js';
|
|
5
5
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
6
6
|
import { MongoReportStorage } from '../MongoReportStorage.js';
|
|
7
7
|
import { PowerSyncMongo } from './db.js';
|
|
@@ -17,6 +17,22 @@ export class MongoStorageProvider {
|
|
|
17
17
|
throw new ServiceAssertionError(`Cannot create MongoDB bucket storage with provided config ${storage.type} !== ${this.type}`);
|
|
18
18
|
}
|
|
19
19
|
const decodedConfig = MongoStorageConfig.decode(storage);
|
|
20
|
+
let objectStorage;
|
|
21
|
+
if (decodedConfig.object_storage?.type === 's3') {
|
|
22
|
+
// Dynamically import S3ObjectStorage to avoid loading AWS SDK unless needed.
|
|
23
|
+
const { S3ObjectStorage } = await import('./v3/object-storage/S3ObjectStorage.js');
|
|
24
|
+
objectStorage = new S3ObjectStorage({
|
|
25
|
+
bucket: decodedConfig.object_storage.bucket,
|
|
26
|
+
region: decodedConfig.object_storage.region,
|
|
27
|
+
prefix: decodedConfig.object_storage.prefix,
|
|
28
|
+
endpoint: decodedConfig.object_storage.endpoint,
|
|
29
|
+
forcePathStyle: decodedConfig.object_storage.force_path_style,
|
|
30
|
+
accessKeyId: decodedConfig.object_storage.access_key_id,
|
|
31
|
+
secretAccessKey: decodedConfig.object_storage.secret_access_key,
|
|
32
|
+
concurrencyLimit: decodedConfig.object_storage.concurrency_limit,
|
|
33
|
+
defaultsMode: decodedConfig.object_storage.defaults_mode
|
|
34
|
+
});
|
|
35
|
+
}
|
|
20
36
|
const client = lib_mongo.db.createMongoClient(decodedConfig, {
|
|
21
37
|
powersyncVersion: POWERSYNC_VERSION,
|
|
22
38
|
maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
|
|
@@ -29,9 +45,25 @@ export class MongoStorageProvider {
|
|
|
29
45
|
// Errors here will cause the process to exit.
|
|
30
46
|
await client.connect();
|
|
31
47
|
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
48
|
+
const readPreference = decodedConfig.bulk_read_preference == null
|
|
49
|
+
? undefined
|
|
50
|
+
: new lib_mongo.mongo.ReadPreference(decodedConfig.bulk_read_preference, undefined, {
|
|
51
|
+
// maxStalenessSeconds is relevant for all modes except 'primary'.
|
|
52
|
+
// 90 is the minimum value.
|
|
53
|
+
maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
|
|
54
|
+
});
|
|
32
55
|
const syncStorageFactory = new MongoBucketStorage(database, {
|
|
33
|
-
|
|
34
|
-
|
|
56
|
+
replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
|
|
57
|
+
readPreference,
|
|
58
|
+
clearBatchThrottleRate: normalizeClearBatchThrottleRate(decodedConfig.clear_batch_throttle_rate),
|
|
59
|
+
checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
|
|
60
|
+
defaultStorageVersion: decodedConfig.default_storage_version,
|
|
61
|
+
// Right now, only MongoDB source databases supports incremental reprocessing.
|
|
62
|
+
// Remove this filter when we support it for other source databases.
|
|
63
|
+
// This assumes a single source connection - revisit if we ever support multiple connections.
|
|
64
|
+
supportsMultipleSyncConfigs: resolvedConfig.connections?.[0]?.type == lib_mongo.MONGO_CONNECTION_TYPE,
|
|
65
|
+
objectStorage,
|
|
66
|
+
inlineThresholdBytes: decodedConfig.object_storage?.inline_threshold_bytes
|
|
35
67
|
});
|
|
36
68
|
// Storage factory for reports
|
|
37
69
|
const reportStorageFactory = new MongoReportStorage(database);
|
|
@@ -43,8 +75,13 @@ export class MongoStorageProvider {
|
|
|
43
75
|
await syncStorageFactory[Symbol.asyncDispose]();
|
|
44
76
|
await client.close();
|
|
45
77
|
},
|
|
46
|
-
tearDown: () => {
|
|
78
|
+
tearDown: async () => {
|
|
47
79
|
logger.info(`Tearing down storage: ${database.db.namespace}...`);
|
|
80
|
+
if (objectStorage != null) {
|
|
81
|
+
logger.info(`Clearing object storage: ${decodedConfig.object_storage.bucket}/${decodedConfig.object_storage.prefix ?? ''}.`);
|
|
82
|
+
const { objectCount } = await objectStorage.deletePrefix('bucket-data/');
|
|
83
|
+
logger.info(`Deleted ${objectCount} objects from object storage.`);
|
|
84
|
+
}
|
|
48
85
|
return database.db.dropDatabase();
|
|
49
86
|
},
|
|
50
87
|
onFatalError: (callback) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoStorageProvider.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoStorageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC3G,OAAO,EAAE,iBAAiB,EAAW,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoStorageProvider.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoStorageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC3G,OAAO,EAAE,iBAAiB,EAAW,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,+BAA+B,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,OAAO,oBAAoB;IAC/B,IAAI,IAAI;QACN,OAAO,SAAS,CAAC,qBAAqB,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAkC;QACjD,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEnC,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC;QACnC,IAAI,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9B,gFAAgF;YAChF,MAAM,IAAI,qBAAqB,CAC7B,6DAA6D,OAAO,CAAC,IAAI,QAAQ,IAAI,CAAC,IAAI,EAAE,CAC7F,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAc,CAAC,CAAC;QAEhE,IAAI,aAAwC,CAAC;QAC7C,IAAI,aAAa,CAAC,cAAc,EAAE,IAAI,KAAK,IAAI,EAAE,CAAC;YAChD,6EAA6E;YAC7E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,wCAAwC,CAAC,CAAC;YACnF,aAAa,GAAG,IAAI,eAAe,CAAC;gBAClC,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,MAAM;gBAC3C,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,MAAM;gBAC3C,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,MAAM;gBAC3C,QAAQ,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ;gBAC/C,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,gBAAgB;gBAC7D,WAAW,EAAE,aAAa,CAAC,cAAc,CAAC,aAAa;gBACvD,eAAe,EAAE,aAAa,CAAC,cAAc,CAAC,iBAAiB;gBAC/D,gBAAgB,EAAE,aAAa,CAAC,cAAc,CAAC,iBAAiB;gBAChE,YAAY,EAAE,aAAa,CAAC,cAAc,CAAC,aAAa;aACzD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC,iBAAiB,CAAC,aAAa,EAAE;YAC3D,gBAAgB,EAAE,iBAAiB;YACnC,WAAW,EAAE,cAAc,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC;SACvD,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,0FAA0F;QAC1F,2FAA2F;QAC3F,yBAAyB;QACzB,8CAA8C;QAC9C,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3F,MAAM,cAAc,GAClB,aAAa,CAAC,oBAAoB,IAAI,IAAI;YACxC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,CAAC,oBAAoB,EAAE,SAAS,EAAE;gBAChF,kEAAkE;gBAClE,2BAA2B;gBAC3B,mBAAmB,EAAE,aAAa,CAAC,oBAAoB,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;aACtF,CAAC,CAAC;QACT,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,QAAQ,EAAE;YAC1D,2BAA2B,EAAE,cAAc,CAAC,gBAAgB;YAC5D,cAAc;YACd,sBAAsB,EAAE,+BAA+B,CAAC,aAAa,CAAC,yBAAyB,CAAC;YAChG,kBAAkB,EAAE,cAAc,CAAC,cAAc,CAAC,8BAA8B,GAAG,MAAM;YACzF,qBAAqB,EAAE,aAAa,CAAC,uBAAuB;YAC5D,8EAA8E;YAC9E,oEAAoE;YACpE,6FAA6F;YAC7F,2BAA2B,EAAE,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,SAAS,CAAC,qBAAqB;YAErG,aAAa;YACb,oBAAoB,EAAE,aAAa,CAAC,cAAc,EAAE,sBAAsB;SAC3E,CAAC,CAAC;QAEH,8BAA8B;QAC9B,MAAM,oBAAoB,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAC9D,OAAO;YACL,OAAO,EAAE,kBAAkB;YAC3B,aAAa,EAAE,oBAAoB;YACnC,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACnB,YAAY,GAAG,IAAI,CAAC;gBACpB,MAAM,kBAAkB,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;gBAChD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;YACD,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACnB,MAAM,CAAC,IAAI,CAAC,yBAAyB,QAAQ,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC;gBACjE,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,MAAM,CAAC,IAAI,CACT,4BAA4B,aAAa,CAAC,cAAe,CAAC,MAAM,IAAI,aAAa,CAAC,cAAe,CAAC,MAAM,IAAI,EAAE,GAAG,CAClH,CAAC;oBACF,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;oBACzE,MAAM,CAAC,IAAI,CAAC,WAAW,WAAW,+BAA+B,CAAC,CAAC;gBACrE,CAAC;gBACD,OAAO,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;YACpC,CAAC;YACD,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACzB,MAAM,CAAC,WAAW,CAAC,gBAAgB,EAAE,GAAG,EAAE;oBACxC,iEAAiE;oBACjE,IAAI,CAAC,YAAY,EAAE,CAAC;wBAClB,yEAAyE;wBACzE,8FAA8F;wBAC9F,kDAAkD;wBAClD,QAAQ,CACN,IAAI,YAAY,CAAC;4BACf,IAAI,EAAE,SAAS,CAAC,WAAW;4BAC3B,WAAW,EAAE,iEAAiE;yBAC/E,CAAC,CACH,CAAC;oBACJ,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;SAC8B,CAAC;IACpC,CAAC;CACF"}
|