@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,13 +1,25 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
|
|
2
|
+
import {
|
|
3
|
+
deserializeParameterLookup,
|
|
4
|
+
JwtPayload,
|
|
5
|
+
SingleSyncConfigBucketDefinitionMapping,
|
|
6
|
+
storage,
|
|
7
|
+
updateSyncRulesFromYaml
|
|
8
|
+
} from '@powersync/service-core';
|
|
2
9
|
import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
|
|
3
|
-
import {
|
|
10
|
+
import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
4
11
|
import * as bson from 'bson';
|
|
5
|
-
import * as sqlite from 'node:sqlite';
|
|
6
12
|
import { describe, expect, test } from 'vitest';
|
|
7
13
|
import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
|
|
14
|
+
import { MongoParsedSyncConfigSet } from '../../src/storage/implementation/MongoParsedSyncConfigSet.js';
|
|
15
|
+
import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementation/MongoPersistedSyncConfigContent.js';
|
|
16
|
+
import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
|
|
8
17
|
import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
|
|
18
|
+
import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
|
|
19
|
+
import { loadBucketDataDocumentV1, serializeBucketDataV1 } from '../../src/storage/implementation/v1/models.js';
|
|
9
20
|
import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
|
|
10
21
|
import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
22
|
+
import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
|
|
11
23
|
import {
|
|
12
24
|
CurrentBucketV3,
|
|
13
25
|
ReplicationStreamDocumentV3,
|
|
@@ -53,17 +65,115 @@ function objectIdGenerator(id: string) {
|
|
|
53
65
|
};
|
|
54
66
|
}
|
|
55
67
|
|
|
56
|
-
|
|
68
|
+
/**
|
|
69
|
+
* Build a standalone parsed sync config set for the given yaml, with its own mapping.
|
|
70
|
+
*
|
|
71
|
+
* This keeps the sync rules and the bucket definition mapping paired, as required by
|
|
72
|
+
* `ResolveTablesOptions.parsedSyncConfig`.
|
|
73
|
+
*/
|
|
74
|
+
function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
|
|
57
75
|
const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
|
|
58
76
|
expect(parsed.errors).toEqual([]);
|
|
59
|
-
return
|
|
77
|
+
return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
|
|
78
|
+
{
|
|
79
|
+
syncConfigId: new bson.ObjectId().toHexString(),
|
|
80
|
+
syncConfig: parsed,
|
|
81
|
+
mapping: SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(parsed)
|
|
82
|
+
}
|
|
83
|
+
]);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory, replicationStreamId: number) {
|
|
87
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
88
|
+
const doc = (await mongoFactory.db.sync_rules.findOne({
|
|
89
|
+
_id: replicationStreamId
|
|
90
|
+
})) as ReplicationStreamDocumentV3 | null;
|
|
91
|
+
if (doc == null) {
|
|
92
|
+
return [];
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const db = mongoFactory.db.versioned(getMongoStorageConfig(doc.storage_version)) as VersionedPowerSyncMongoV3;
|
|
96
|
+
const syncConfigDocs = await db.syncConfigDefinitions
|
|
97
|
+
.find({
|
|
98
|
+
_id: { $in: doc.sync_configs.map((config) => config._id) }
|
|
99
|
+
})
|
|
100
|
+
.toArray();
|
|
101
|
+
|
|
102
|
+
return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
|
|
60
103
|
}
|
|
61
104
|
|
|
105
|
+
/**
|
|
106
|
+
* Get a MongoDB V3 storage bucket definition id, from a bucket name.
|
|
107
|
+
*
|
|
108
|
+
* This relies on internals of bucket naming, but helps for simplifying tests.
|
|
109
|
+
*/
|
|
110
|
+
function bucketDefinitionId(bucket: string) {
|
|
111
|
+
const match = bucket.match(/^\w+\|\w+\.\w+\.(\w+)\[/);
|
|
112
|
+
if (match == null) {
|
|
113
|
+
throw new Error(`Expected versioned bucket name, got ${bucket}`);
|
|
114
|
+
}
|
|
115
|
+
return parseInt(match[1], 16);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
test('V1 bucket data retains an optional persisted subkey', () => {
|
|
119
|
+
const sourceTable = new bson.ObjectId();
|
|
120
|
+
const document = serializeBucketDataV1({
|
|
121
|
+
bucketKey: {
|
|
122
|
+
replicationStreamId: 1,
|
|
123
|
+
definitionId: '0',
|
|
124
|
+
bucket: 'global[]'
|
|
125
|
+
},
|
|
126
|
+
o: 1n,
|
|
127
|
+
op: 'PUT',
|
|
128
|
+
source_table: sourceTable,
|
|
129
|
+
source_key: 'source-key',
|
|
130
|
+
subkey: 'persisted-subkey',
|
|
131
|
+
table: 'items',
|
|
132
|
+
row_id: 'item-1',
|
|
133
|
+
checksum: 1n,
|
|
134
|
+
data: '{"id":"item-1"}'
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
expect(loadBucketDataDocumentV1(document).subkey).toBe('persisted-subkey');
|
|
138
|
+
|
|
139
|
+
const { subkey: _subkey, ...legacyDocument } = document;
|
|
140
|
+
expect(loadBucketDataDocumentV1(legacyDocument).subkey).toBeUndefined();
|
|
141
|
+
});
|
|
142
|
+
|
|
62
143
|
function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
|
|
63
144
|
register.registerSyncTests(storageConfig.factory, {
|
|
64
145
|
storageVersion,
|
|
65
146
|
tableIdStrings: storageConfig.tableIdStrings
|
|
66
147
|
});
|
|
148
|
+
|
|
149
|
+
test('updates source metadata on an existing resolved table', async () => {
|
|
150
|
+
await using factory = await storageConfig.factory();
|
|
151
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
152
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
153
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
154
|
+
const source = sourceDescriptor('test');
|
|
155
|
+
|
|
156
|
+
const initial = await writer.resolveTables({ connection_id: 1, source });
|
|
157
|
+
expect(initial.tables[0].sourceMetadata).toBeNull();
|
|
158
|
+
|
|
159
|
+
const sourceMetadata = { captureTableObjectId: 42 };
|
|
160
|
+
const updated = await writer.resolveTables({
|
|
161
|
+
connection_id: 1,
|
|
162
|
+
source,
|
|
163
|
+
reconcileSourceTables: ({ candidates }) => ({
|
|
164
|
+
compatibleTables: candidates.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
|
|
165
|
+
incompatibleTables: [],
|
|
166
|
+
newTableValues: { sourceMetadata }
|
|
167
|
+
})
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
expect(updated.tables.map((table) => table.id.toString())).toEqual(
|
|
171
|
+
initial.tables.map((table) => table.id.toString())
|
|
172
|
+
);
|
|
173
|
+
expect(updated.tables[0].sourceMetadata).toEqual(sourceMetadata);
|
|
174
|
+
expect((await writer.getSourceTableStatus(updated.tables[0]))?.sourceMetadata).toEqual(sourceMetadata);
|
|
175
|
+
});
|
|
176
|
+
|
|
67
177
|
// The split of returned results can vary depending on storage drivers
|
|
68
178
|
test('large batch (2)', async () => {
|
|
69
179
|
// Test syncing a batch of data that is small in count,
|
|
@@ -82,6 +192,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
82
192
|
)
|
|
83
193
|
);
|
|
84
194
|
const bucketStorage = factory.getInstance(syncRules);
|
|
195
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
85
196
|
|
|
86
197
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
87
198
|
|
|
@@ -135,8 +246,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
135
246
|
const checkpoint = flushResult!.flushed_op;
|
|
136
247
|
|
|
137
248
|
const options: storage.BucketDataBatchOptions = {};
|
|
138
|
-
const batch1 = await test_utils.
|
|
139
|
-
bucketStorage.getBucketDataBatch(
|
|
249
|
+
const batch1 = await test_utils.getBatchArray(
|
|
250
|
+
bucketStorage.getBucketDataBatch(
|
|
251
|
+
test_utils.testCheckpoint(checkpoint),
|
|
252
|
+
[bucketRequest(syncRulesContent, 'global[]', 0n)],
|
|
253
|
+
options
|
|
254
|
+
)
|
|
140
255
|
);
|
|
141
256
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
142
257
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
@@ -148,10 +263,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
148
263
|
next_after: '2'
|
|
149
264
|
});
|
|
150
265
|
|
|
151
|
-
const batch2 = await test_utils.
|
|
266
|
+
const batch2 = await test_utils.getBatchArray(
|
|
152
267
|
bucketStorage.getBucketDataBatch(
|
|
153
|
-
checkpoint,
|
|
154
|
-
[bucketRequest(
|
|
268
|
+
test_utils.testCheckpoint(checkpoint),
|
|
269
|
+
[bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
|
|
155
270
|
options
|
|
156
271
|
)
|
|
157
272
|
);
|
|
@@ -164,10 +279,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
164
279
|
next_after: '3'
|
|
165
280
|
});
|
|
166
281
|
|
|
167
|
-
const batch3 = await test_utils.
|
|
282
|
+
const batch3 = await test_utils.getBatchArray(
|
|
168
283
|
bucketStorage.getBucketDataBatch(
|
|
169
|
-
checkpoint,
|
|
170
|
-
[bucketRequest(
|
|
284
|
+
test_utils.testCheckpoint(checkpoint),
|
|
285
|
+
[bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
|
|
171
286
|
options
|
|
172
287
|
)
|
|
173
288
|
);
|
|
@@ -189,7 +304,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
189
304
|
await Promise.all(
|
|
190
305
|
(
|
|
191
306
|
await mongoFactory.db.db
|
|
192
|
-
.listCollections(
|
|
307
|
+
.listCollections(
|
|
308
|
+
{ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) },
|
|
309
|
+
{ nameOnly: true }
|
|
310
|
+
)
|
|
193
311
|
.toArray()
|
|
194
312
|
).map((collection: { name: string }) =>
|
|
195
313
|
mongoFactory.db.db
|
|
@@ -388,8 +506,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
388
506
|
}
|
|
389
507
|
);
|
|
390
508
|
|
|
391
|
-
test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions
|
|
392
|
-
// Tests the behavior of resolveTables when bucket data sources and parameter index creators are added
|
|
509
|
+
test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions', async () => {
|
|
510
|
+
// Tests the behavior of resolveTables when bucket data sources and parameter index creators are added.
|
|
393
511
|
// These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
|
|
394
512
|
// This just tests the specific resolveTables behavior.
|
|
395
513
|
|
|
@@ -434,20 +552,20 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
434
552
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
|
|
435
553
|
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
436
554
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
437
|
-
const fullRules =
|
|
438
|
-
const dataOnlyRules =
|
|
439
|
-
const parameterOnlyRules =
|
|
440
|
-
const eventOnlyRules =
|
|
555
|
+
const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
|
|
556
|
+
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
|
|
557
|
+
const parameterOnlyRules = parsedSyncConfigSetFor(parameterOnlyRulesYaml, storageVersion);
|
|
558
|
+
const eventOnlyRules = parsedSyncConfigSetFor(eventOnlyRulesYaml, storageVersion);
|
|
441
559
|
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
442
560
|
const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
|
|
443
561
|
const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
|
|
444
|
-
const
|
|
562
|
+
const addedEventTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
|
|
445
563
|
|
|
446
564
|
const dataOnly = await writer.resolveTables({
|
|
447
565
|
connection_id: 1,
|
|
448
566
|
source,
|
|
449
567
|
idGenerator: () => dataOnlyTableId,
|
|
450
|
-
|
|
568
|
+
parsedSyncConfig: dataOnlyRules
|
|
451
569
|
});
|
|
452
570
|
expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
453
571
|
expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
@@ -458,7 +576,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
458
576
|
connection_id: 1,
|
|
459
577
|
source,
|
|
460
578
|
idGenerator: () => addedParameterTableId,
|
|
461
|
-
|
|
579
|
+
parsedSyncConfig: fullRules
|
|
462
580
|
});
|
|
463
581
|
// Adding a definition always creates a new SourceTable
|
|
464
582
|
expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
|
|
@@ -472,41 +590,40 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
472
590
|
idGenerator: () => {
|
|
473
591
|
throw new Error('data-only resolve should reuse existing v3 source table');
|
|
474
592
|
},
|
|
475
|
-
|
|
593
|
+
parsedSyncConfig: dataOnlyRules
|
|
476
594
|
});
|
|
477
595
|
expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
478
|
-
//
|
|
479
|
-
expect(removedParameter.dropTables.map((table) => table.id)).toEqual([
|
|
596
|
+
// Same-identity tables that no longer cover desired memberships are left in place.
|
|
597
|
+
expect(removedParameter.dropTables.map((table) => table.id)).toEqual([]);
|
|
480
598
|
expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
|
|
481
599
|
expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
|
|
482
|
-
await writer.drop(removedParameter.dropTables);
|
|
483
600
|
|
|
484
601
|
const removedData = await writer.resolveTables({
|
|
485
602
|
connection_id: 1,
|
|
486
603
|
source,
|
|
487
|
-
idGenerator: () =>
|
|
488
|
-
|
|
604
|
+
idGenerator: () => {
|
|
605
|
+
throw new Error('parameter-only resolve should reuse existing v3 source table');
|
|
606
|
+
},
|
|
607
|
+
parsedSyncConfig: parameterOnlyRules
|
|
489
608
|
});
|
|
490
609
|
|
|
491
|
-
// This goes from dataOnlyRules -> parameterOnlyRules, which
|
|
492
|
-
//
|
|
493
|
-
expect(removedData.tables.map((table) => table.id)).toEqual([
|
|
494
|
-
expect(removedData.dropTables.map((table) => table.id)).toEqual([
|
|
610
|
+
// This goes from dataOnlyRules -> parameterOnlyRules, which removes one definition but
|
|
611
|
+
// reuses the existing parameter-only source table.
|
|
612
|
+
expect(removedData.tables.map((table) => table.id)).toEqual([addedParameterTableId]);
|
|
613
|
+
expect(removedData.dropTables.map((table) => table.id)).toEqual([]);
|
|
495
614
|
expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
|
|
496
615
|
expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
|
|
497
|
-
await writer.drop(removedData.dropTables);
|
|
498
616
|
|
|
499
617
|
const eventOnly = await writer.resolveTables({
|
|
500
618
|
connection_id: 1,
|
|
501
619
|
source,
|
|
502
|
-
idGenerator: () =>
|
|
503
|
-
|
|
504
|
-
},
|
|
505
|
-
syncRules: eventOnlyRules
|
|
620
|
+
idGenerator: () => addedEventTableId,
|
|
621
|
+
parsedSyncConfig: eventOnlyRules
|
|
506
622
|
});
|
|
507
623
|
|
|
508
|
-
//
|
|
509
|
-
expect(eventOnly.tables
|
|
624
|
+
// A newly-added event gets its own source table so that its existing rows are snapshotted.
|
|
625
|
+
expect(eventOnly.tables).toHaveLength(1);
|
|
626
|
+
expect(eventOnly.tables[0].id).toEqual(addedEventTableId);
|
|
510
627
|
expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
511
628
|
expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
|
|
512
629
|
expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
|
|
@@ -515,6 +632,234 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
515
632
|
expect(eventOnly.tables[0].syncEvent).toBe(true);
|
|
516
633
|
});
|
|
517
634
|
|
|
635
|
+
test.runIf(storageVersion >= 3)('resolveTables assigns an event definition to one split source table', async () => {
|
|
636
|
+
// When memberships are split over multiple SourceTables for the same ref, a row change
|
|
637
|
+
// is saved once per table. Each event-definition id must belong to only one of those tables,
|
|
638
|
+
// otherwise that definition would be evaluated more than once for the same row change.
|
|
639
|
+
// Other tables may independently own and evaluate different event definitions.
|
|
640
|
+
const dataOnlyEventYaml = `
|
|
641
|
+
bucket_definitions:
|
|
642
|
+
by_owner:
|
|
643
|
+
parameters:
|
|
644
|
+
- SELECT token_parameters.owner_id as owner_id
|
|
645
|
+
data:
|
|
646
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
647
|
+
|
|
648
|
+
event_definitions:
|
|
649
|
+
write_checkpoints:
|
|
650
|
+
payloads:
|
|
651
|
+
- SELECT id, owner_id FROM memberships
|
|
652
|
+
`;
|
|
653
|
+
const fullEventYaml = `
|
|
654
|
+
bucket_definitions:
|
|
655
|
+
by_owner:
|
|
656
|
+
parameters:
|
|
657
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
658
|
+
data:
|
|
659
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
660
|
+
|
|
661
|
+
event_definitions:
|
|
662
|
+
write_checkpoints:
|
|
663
|
+
payloads:
|
|
664
|
+
- SELECT id, owner_id FROM memberships
|
|
665
|
+
`;
|
|
666
|
+
|
|
667
|
+
await using factory = await storageConfig.factory();
|
|
668
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
|
|
669
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
670
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
671
|
+
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
|
|
672
|
+
const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
|
|
673
|
+
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
674
|
+
|
|
675
|
+
const initial = await writer.resolveTables({
|
|
676
|
+
connection_id: 1,
|
|
677
|
+
source,
|
|
678
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
|
|
679
|
+
parsedSyncConfig: dataOnlyRules
|
|
680
|
+
});
|
|
681
|
+
expect(initial.tables).toHaveLength(1);
|
|
682
|
+
expect(initial.tables[0].syncEvent).toBe(true);
|
|
683
|
+
|
|
684
|
+
// Adding the table-based parameter lookup creates a second SourceTable for the same ref.
|
|
685
|
+
const split = await writer.resolveTables({
|
|
686
|
+
connection_id: 1,
|
|
687
|
+
source,
|
|
688
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
|
|
689
|
+
parsedSyncConfig: fullRules
|
|
690
|
+
});
|
|
691
|
+
expect(split.tables).toHaveLength(2);
|
|
692
|
+
// Both tables match the event by ref, but this event id belongs to only one of them.
|
|
693
|
+
const eventOwners = split.tables.filter((table) => table.syncEvent);
|
|
694
|
+
expect(eventOwners).toHaveLength(1);
|
|
695
|
+
|
|
696
|
+
// getSourceTableStatus rehydrates the persisted event memberships rather than recomputing
|
|
697
|
+
// them from the ref, so refreshing the other table does not assign this event to it.
|
|
698
|
+
const nonOwner = split.tables.find((table) => !table.syncEvent)!;
|
|
699
|
+
const refreshed = await writer.getSourceTableStatus(nonOwner);
|
|
700
|
+
expect(refreshed!.syncEvent).toBe(false);
|
|
701
|
+
});
|
|
702
|
+
|
|
703
|
+
test.runIf(storageVersion >= 3)(
|
|
704
|
+
'reuses an unchanged event definition without resnapshotting or duplicate evaluation',
|
|
705
|
+
async () => {
|
|
706
|
+
const firstYaml = `
|
|
707
|
+
config:
|
|
708
|
+
edition: 3
|
|
709
|
+
|
|
710
|
+
streams:
|
|
711
|
+
by_owner:
|
|
712
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
713
|
+
|
|
714
|
+
event_definitions:
|
|
715
|
+
write_checkpoints:
|
|
716
|
+
payloads:
|
|
717
|
+
- SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0
|
|
718
|
+
`;
|
|
719
|
+
const secondYaml = `
|
|
720
|
+
config:
|
|
721
|
+
edition: 3
|
|
722
|
+
|
|
723
|
+
streams:
|
|
724
|
+
by_owner:
|
|
725
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
726
|
+
by_project:
|
|
727
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
728
|
+
|
|
729
|
+
event_definitions:
|
|
730
|
+
write_checkpoints:
|
|
731
|
+
payloads:
|
|
732
|
+
- select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0
|
|
733
|
+
`;
|
|
734
|
+
|
|
735
|
+
await using factory = await storageConfig.factory();
|
|
736
|
+
const emittedEvents: { name: string; eventId: string | undefined }[] = [];
|
|
737
|
+
const disposeListener = factory.registerListener({
|
|
738
|
+
replicationEvent: ({ event, event_id }) => emittedEvents.push({ name: event.name, eventId: event_id })
|
|
739
|
+
});
|
|
740
|
+
try {
|
|
741
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion }));
|
|
742
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
743
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
744
|
+
const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
|
|
745
|
+
const firstResolved = await firstWriter.resolveTables({
|
|
746
|
+
connection_id: 1,
|
|
747
|
+
source,
|
|
748
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38351')
|
|
749
|
+
});
|
|
750
|
+
const eventId = [...firstResolved.tables[0].eventDefinitionIds!][0];
|
|
751
|
+
expect(eventId).toBeDefined();
|
|
752
|
+
await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1');
|
|
753
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
754
|
+
await firstWriter.commit('1/1');
|
|
755
|
+
|
|
756
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(secondYaml, { storageVersion }));
|
|
757
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
758
|
+
const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage;
|
|
759
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
760
|
+
const resolved = await secondWriter.resolveTables({
|
|
761
|
+
connection_id: 1,
|
|
762
|
+
source,
|
|
763
|
+
idGenerator: () => {
|
|
764
|
+
throw new Error('unchanged event definition should reuse the snapshotted source table');
|
|
765
|
+
}
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
expect(resolved.tables).toHaveLength(1);
|
|
769
|
+
expect(resolved.tables[0].id).toEqual(firstResolved.tables[0].id);
|
|
770
|
+
expect(resolved.tables[0].snapshotComplete).toBe(true);
|
|
771
|
+
expect([...resolved.tables[0].eventDefinitionIds!]).toEqual([eventId]);
|
|
772
|
+
|
|
773
|
+
emittedEvents.length = 0;
|
|
774
|
+
await secondWriter.save({
|
|
775
|
+
sourceTable: resolved.tables[0],
|
|
776
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
777
|
+
after: { id: 'checkpoint-1', user_id: 'user-1', checkpoint: 1n, active: 1 },
|
|
778
|
+
afterReplicaId: test_utils.rid('checkpoint-1')
|
|
779
|
+
});
|
|
780
|
+
expect(emittedEvents).toEqual([{ name: 'write_checkpoints', eventId }]);
|
|
781
|
+
} finally {
|
|
782
|
+
disposeListener();
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
);
|
|
786
|
+
|
|
787
|
+
test.runIf(storageVersion >= 3)('creates snapshot work for a changed event definition', async () => {
|
|
788
|
+
const yaml = (active: boolean) => `
|
|
789
|
+
config:
|
|
790
|
+
edition: 3
|
|
791
|
+
|
|
792
|
+
streams:
|
|
793
|
+
by_owner:
|
|
794
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
795
|
+
|
|
796
|
+
event_definitions:
|
|
797
|
+
write_checkpoints:
|
|
798
|
+
payloads:
|
|
799
|
+
- SELECT user_id, checkpoint FROM checkpoints WHERE active = ${active}
|
|
800
|
+
`;
|
|
801
|
+
|
|
802
|
+
await using factory = await storageConfig.factory();
|
|
803
|
+
const emittedEvents: { name: string; eventId: string | undefined }[] = [];
|
|
804
|
+
const disposeListener = factory.registerListener({
|
|
805
|
+
replicationEvent: ({ event, event_id }) => emittedEvents.push({ name: event.name, eventId: event_id })
|
|
806
|
+
});
|
|
807
|
+
try {
|
|
808
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion }));
|
|
809
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
810
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
811
|
+
const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
|
|
812
|
+
const firstResolved = await firstWriter.resolveTables({
|
|
813
|
+
connection_id: 1,
|
|
814
|
+
source,
|
|
815
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38352')
|
|
816
|
+
});
|
|
817
|
+
const oldEventId = [...firstResolved.tables[0].eventDefinitionIds!][0];
|
|
818
|
+
await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1');
|
|
819
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
820
|
+
await firstWriter.commit('1/1');
|
|
821
|
+
|
|
822
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(false), { storageVersion }));
|
|
823
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
824
|
+
const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage;
|
|
825
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
826
|
+
const resolved = await secondWriter.resolveTables({
|
|
827
|
+
connection_id: 1,
|
|
828
|
+
source,
|
|
829
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38353')
|
|
830
|
+
});
|
|
831
|
+
|
|
832
|
+
expect(resolved.tables).toHaveLength(2);
|
|
833
|
+
const oldTable = resolved.tables.find((table) => table.eventDefinitionIds?.has(oldEventId))!;
|
|
834
|
+
const newTable = resolved.tables.find((table) => !table.eventDefinitionIds?.has(oldEventId))!;
|
|
835
|
+
const newEventId = [...newTable.eventDefinitionIds!][0];
|
|
836
|
+
expect(newEventId).toBeDefined();
|
|
837
|
+
expect(newEventId).not.toBe(oldEventId);
|
|
838
|
+
expect(oldTable.snapshotComplete).toBe(true);
|
|
839
|
+
expect(newTable.snapshotComplete).toBe(false);
|
|
840
|
+
expect(newTable.bucketDataSources).toEqual([]);
|
|
841
|
+
expect(newTable.parameterLookupSources).toEqual([]);
|
|
842
|
+
|
|
843
|
+
await expect(secondWriter.markSnapshotDone('2/1')).rejects.toThrow(
|
|
844
|
+
'Cannot mark snapshot done while source tables still require snapshotting'
|
|
845
|
+
);
|
|
846
|
+
|
|
847
|
+
emittedEvents.length = 0;
|
|
848
|
+
await secondWriter.save({
|
|
849
|
+
sourceTable: newTable,
|
|
850
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
851
|
+
after: { id: 'checkpoint-2', user_id: 'user-2', checkpoint: 2n, active: 0 },
|
|
852
|
+
afterReplicaId: test_utils.rid('checkpoint-2')
|
|
853
|
+
});
|
|
854
|
+
expect(emittedEvents).toEqual([{ name: 'write_checkpoints', eventId: newEventId }]);
|
|
855
|
+
|
|
856
|
+
await secondWriter.markTableSnapshotDone([newTable], '2/1');
|
|
857
|
+
await secondWriter.markSnapshotDone('2/1');
|
|
858
|
+
} finally {
|
|
859
|
+
disposeListener();
|
|
860
|
+
}
|
|
861
|
+
});
|
|
862
|
+
|
|
518
863
|
test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
|
|
519
864
|
await using factory = await storageConfig.factory();
|
|
520
865
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -531,7 +876,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
531
876
|
)
|
|
532
877
|
);
|
|
533
878
|
const bucketStorage = factory.getInstance(syncRules);
|
|
534
|
-
const
|
|
879
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
880
|
+
const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
|
|
535
881
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
536
882
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
537
883
|
|
|
@@ -552,7 +898,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
552
898
|
const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
|
|
553
899
|
const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
|
|
554
900
|
const buckets = await querier.queryDynamicBucketDescriptions({
|
|
555
|
-
async getParameterSets(lookups) {
|
|
901
|
+
async getParameterSets(lookups: ScopedParameterLookup[]) {
|
|
556
902
|
expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
|
|
557
903
|
expect(lookups[0].indexId).toEqual('1');
|
|
558
904
|
|
|
@@ -561,33 +907,729 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
561
907
|
return parameter_sets;
|
|
562
908
|
}
|
|
563
909
|
});
|
|
564
|
-
expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(
|
|
910
|
+
expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]);
|
|
565
911
|
|
|
566
912
|
const mongoFactory = factory as MongoBucketStorage;
|
|
567
913
|
const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
|
|
568
|
-
const currentDataCollections = await db.
|
|
914
|
+
const currentDataCollections = await db.listSourceRecordCollections(syncRules.replicationStreamId);
|
|
569
915
|
const currentData = await currentDataCollections[0]?.findOne({});
|
|
570
916
|
const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
|
|
571
917
|
expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
|
|
572
918
|
|
|
573
919
|
const bucketCollections = await mongoFactory.db.db
|
|
574
|
-
.listCollections({ name: new RegExp(`^bucket_data_${syncRules.
|
|
920
|
+
.listCollections({ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
|
|
575
921
|
.toArray();
|
|
576
922
|
expect(
|
|
577
|
-
bucketCollections.some(
|
|
923
|
+
bucketCollections.some(
|
|
924
|
+
(collection) => collection.name === `bucket_data_${syncRules.replicationStreamId}_${firstBucket?.def}`
|
|
925
|
+
)
|
|
578
926
|
).toBe(true);
|
|
579
927
|
|
|
580
|
-
const syncRule = (await mongoFactory.db.sync_rules.findOne({
|
|
928
|
+
const syncRule = (await mongoFactory.db.sync_rules.findOne({
|
|
929
|
+
_id: syncRules.replicationStreamId
|
|
930
|
+
})) as ReplicationStreamDocumentV3;
|
|
581
931
|
const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
|
|
582
932
|
const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
|
|
583
933
|
expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
|
|
584
934
|
|
|
585
935
|
const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
|
|
586
936
|
expect(parameterIndexId).toBeDefined();
|
|
587
|
-
const parameterEntry = await db.
|
|
937
|
+
const parameterEntry = await db.parameterIndex(syncRules.replicationStreamId, parameterIndexId!).findOne({});
|
|
588
938
|
expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
|
|
589
939
|
});
|
|
590
940
|
|
|
941
|
+
test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => {
|
|
942
|
+
await using factory = await storageConfig.factory();
|
|
943
|
+
|
|
944
|
+
const first = await factory.updateSyncRules(
|
|
945
|
+
updateSyncRulesFromYaml(
|
|
946
|
+
`
|
|
947
|
+
config:
|
|
948
|
+
edition: 2
|
|
949
|
+
|
|
950
|
+
streams:
|
|
951
|
+
by_owner:
|
|
952
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
953
|
+
`,
|
|
954
|
+
{ storageVersion }
|
|
955
|
+
)
|
|
956
|
+
);
|
|
957
|
+
const second = await factory.updateSyncRules(
|
|
958
|
+
updateSyncRulesFromYaml(
|
|
959
|
+
`
|
|
960
|
+
config:
|
|
961
|
+
edition: 2
|
|
962
|
+
|
|
963
|
+
streams:
|
|
964
|
+
by_project:
|
|
965
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
966
|
+
`,
|
|
967
|
+
{ storageVersion }
|
|
968
|
+
)
|
|
969
|
+
);
|
|
970
|
+
|
|
971
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
972
|
+
expect(
|
|
973
|
+
(await factory.getStoppedReplicationStreams()).find(
|
|
974
|
+
(stream) => stream.replicationStreamId == first.replicationStreamId
|
|
975
|
+
)?.state
|
|
976
|
+
).toBe(storage.SyncRuleState.STOP);
|
|
977
|
+
|
|
978
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
979
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
980
|
+
expect(replicatingStreams[0].replicationStreamId).toEqual(second.replicationStreamId);
|
|
981
|
+
|
|
982
|
+
const configs = second.syncConfigContent;
|
|
983
|
+
expect(configs).toHaveLength(1);
|
|
984
|
+
const statuses = await Promise.all(configs.map((config) => config.getSyncConfigStatus()));
|
|
985
|
+
expect(statuses.map((status) => status?.state)).toEqual([storage.SyncRuleState.PROCESSING]);
|
|
986
|
+
expect(statuses.map((status) => status?.id)).toEqual(configs.map((config) => config.syncConfigId));
|
|
987
|
+
const parsed = replicatingStreams[0].parsed(test_utils.PARSE_OPTIONS);
|
|
988
|
+
expect(parsed.syncConfigs).toHaveLength(1);
|
|
989
|
+
expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
|
|
990
|
+
|
|
991
|
+
const bucketStorage = factory.getInstance(replicatingStreams[0]);
|
|
992
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
993
|
+
const resolved = await writer.resolveTables({
|
|
994
|
+
connection_id: 1,
|
|
995
|
+
source: sourceDescriptor('todos', { objectId: 'todos-relation' }),
|
|
996
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834b')
|
|
997
|
+
});
|
|
998
|
+
expect(resolved.tables).toHaveLength(1);
|
|
999
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
|
|
1000
|
+
});
|
|
1001
|
+
|
|
1002
|
+
test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => {
|
|
1003
|
+
await using factory = await storageConfig.factory();
|
|
1004
|
+
|
|
1005
|
+
const first = await factory.updateSyncRules(
|
|
1006
|
+
updateSyncRulesFromYaml(
|
|
1007
|
+
`
|
|
1008
|
+
config:
|
|
1009
|
+
edition: 3
|
|
1010
|
+
|
|
1011
|
+
streams:
|
|
1012
|
+
by_owner:
|
|
1013
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1014
|
+
`,
|
|
1015
|
+
{ storageVersion }
|
|
1016
|
+
)
|
|
1017
|
+
);
|
|
1018
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1019
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1020
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1021
|
+
await firstWriter.commit('1/1');
|
|
1022
|
+
|
|
1023
|
+
const second = await factory.updateSyncRules(
|
|
1024
|
+
updateSyncRulesFromYaml(
|
|
1025
|
+
`
|
|
1026
|
+
config:
|
|
1027
|
+
edition: 3
|
|
1028
|
+
|
|
1029
|
+
streams:
|
|
1030
|
+
by_project:
|
|
1031
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1032
|
+
`,
|
|
1033
|
+
{ storageVersion }
|
|
1034
|
+
)
|
|
1035
|
+
);
|
|
1036
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1037
|
+
|
|
1038
|
+
const bucketStorage = factory.getInstance(second);
|
|
1039
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1040
|
+
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
1041
|
+
|
|
1042
|
+
const resolved = await writer.resolveTables({
|
|
1043
|
+
connection_id: 1,
|
|
1044
|
+
source,
|
|
1045
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834c')
|
|
1046
|
+
});
|
|
1047
|
+
expect(resolved.tables).toHaveLength(1);
|
|
1048
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(2);
|
|
1049
|
+
|
|
1050
|
+
await writer.markAllSnapshotDone('2/1');
|
|
1051
|
+
await writer.commit('2/1');
|
|
1052
|
+
|
|
1053
|
+
const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1054
|
+
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1055
|
+
const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
|
|
1056
|
+
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
1057
|
+
});
|
|
1058
|
+
|
|
1059
|
+
test.runIf(storageVersion >= 3)(
|
|
1060
|
+
'editing a stream query on the same table does not drop the active source table',
|
|
1061
|
+
async () => {
|
|
1062
|
+
await using factory = await storageConfig.factory();
|
|
1063
|
+
|
|
1064
|
+
const first = await factory.updateSyncRules(
|
|
1065
|
+
updateSyncRulesFromYaml(
|
|
1066
|
+
`
|
|
1067
|
+
config:
|
|
1068
|
+
edition: 3
|
|
1069
|
+
|
|
1070
|
+
streams:
|
|
1071
|
+
todos:
|
|
1072
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1073
|
+
`,
|
|
1074
|
+
{ storageVersion }
|
|
1075
|
+
)
|
|
1076
|
+
);
|
|
1077
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1078
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1079
|
+
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
1080
|
+
const firstResolved = await firstWriter.resolveTables({
|
|
1081
|
+
connection_id: 1,
|
|
1082
|
+
source,
|
|
1083
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
|
|
1084
|
+
});
|
|
1085
|
+
expect(firstResolved.dropTables).toHaveLength(0);
|
|
1086
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1087
|
+
await firstWriter.commit('1/1');
|
|
1088
|
+
|
|
1089
|
+
const second = await factory.updateSyncRules(
|
|
1090
|
+
updateSyncRulesFromYaml(
|
|
1091
|
+
`
|
|
1092
|
+
config:
|
|
1093
|
+
edition: 3
|
|
1094
|
+
|
|
1095
|
+
streams:
|
|
1096
|
+
todos:
|
|
1097
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1098
|
+
`,
|
|
1099
|
+
{ storageVersion }
|
|
1100
|
+
)
|
|
1101
|
+
);
|
|
1102
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1103
|
+
|
|
1104
|
+
const deploying = await factory.getDeployingSyncConfig();
|
|
1105
|
+
expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
|
|
1106
|
+
const bucketStorage = deploying!.storage;
|
|
1107
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1108
|
+
const resolved = await writer.resolveTables({
|
|
1109
|
+
connection_id: 1,
|
|
1110
|
+
source,
|
|
1111
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834e')
|
|
1112
|
+
});
|
|
1113
|
+
|
|
1114
|
+
expect(resolved.dropTables).toHaveLength(0);
|
|
1115
|
+
expect(resolved.tables.map((table) => table.id.toString())).toEqual(['6544e3899293153fa7b3834e']);
|
|
1116
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
|
|
1117
|
+
|
|
1118
|
+
const activeStatus = await firstWriter.getSourceTableStatus(firstResolved.tables[0]);
|
|
1119
|
+
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
1120
|
+
}
|
|
1121
|
+
);
|
|
1122
|
+
|
|
1123
|
+
test.runIf(storageVersion >= 3)('reserves historical mapping ids without reusing stopped configs', async () => {
|
|
1124
|
+
await using factory = await storageConfig.factory();
|
|
1125
|
+
|
|
1126
|
+
const ownerRules = `
|
|
1127
|
+
config:
|
|
1128
|
+
edition: 3
|
|
1129
|
+
|
|
1130
|
+
streams:
|
|
1131
|
+
by_owner:
|
|
1132
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1133
|
+
`;
|
|
1134
|
+
const projectRules = `
|
|
1135
|
+
config:
|
|
1136
|
+
edition: 3
|
|
1137
|
+
|
|
1138
|
+
streams:
|
|
1139
|
+
by_project:
|
|
1140
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1141
|
+
`;
|
|
1142
|
+
const statusRules = `
|
|
1143
|
+
config:
|
|
1144
|
+
edition: 3
|
|
1145
|
+
|
|
1146
|
+
streams:
|
|
1147
|
+
by_status:
|
|
1148
|
+
query: SELECT * FROM todos WHERE status = subscription.parameter('status')
|
|
1149
|
+
`;
|
|
1150
|
+
|
|
1151
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
|
|
1152
|
+
const firstStorage = factory.getInstance(first);
|
|
1153
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1154
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1155
|
+
await firstWriter.commit('1/1');
|
|
1156
|
+
|
|
1157
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(projectRules, { storageVersion }));
|
|
1158
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1159
|
+
|
|
1160
|
+
const third = await factory.updateSyncRules(updateSyncRulesFromYaml(statusRules, { storageVersion }));
|
|
1161
|
+
expect(third.replicationStreamId).toBe(first.replicationStreamId);
|
|
1162
|
+
|
|
1163
|
+
const configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1164
|
+
const statuses = await Promise.all(
|
|
1165
|
+
configs.map(async (config) => [config, await config.getSyncConfigStatus()] as const)
|
|
1166
|
+
);
|
|
1167
|
+
// ownerRules
|
|
1168
|
+
const activeConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.ACTIVE)?.[0];
|
|
1169
|
+
// projectRules
|
|
1170
|
+
const stoppedConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.STOP)?.[0];
|
|
1171
|
+
// statusRules
|
|
1172
|
+
const processingConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.PROCESSING)?.[0];
|
|
1173
|
+
expect(stoppedConfig).toBeDefined();
|
|
1174
|
+
expect(activeConfig).toBeDefined();
|
|
1175
|
+
expect(processingConfig).toBeDefined();
|
|
1176
|
+
|
|
1177
|
+
const activePrefix = bucketRequest(activeConfig!, 'by_owner|0["owner"]').bucket;
|
|
1178
|
+
const stoppedPrefix = bucketRequest(stoppedConfig!, 'by_project|0["project"]').bucket;
|
|
1179
|
+
const processingPrefix = bucketRequest(processingConfig!, 'by_status|0["status"]').bucket;
|
|
1180
|
+
|
|
1181
|
+
expect(processingPrefix).not.toBe(stoppedPrefix);
|
|
1182
|
+
expect(processingPrefix).not.toBe(activePrefix);
|
|
1183
|
+
expect(bucketDefinitionId(processingPrefix)).toBeGreaterThan(
|
|
1184
|
+
Math.max(bucketDefinitionId(stoppedPrefix), bucketDefinitionId(activePrefix))
|
|
1185
|
+
);
|
|
1186
|
+
});
|
|
1187
|
+
|
|
1188
|
+
test.runIf(storageVersion >= 3)('table snapshot status only affects sync configs using that table', async () => {
|
|
1189
|
+
await using factory = await storageConfig.factory();
|
|
1190
|
+
|
|
1191
|
+
const first = await factory.updateSyncRules(
|
|
1192
|
+
updateSyncRulesFromYaml(
|
|
1193
|
+
`
|
|
1194
|
+
config:
|
|
1195
|
+
edition: 3
|
|
1196
|
+
|
|
1197
|
+
streams:
|
|
1198
|
+
by_owner:
|
|
1199
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1200
|
+
`,
|
|
1201
|
+
{ storageVersion }
|
|
1202
|
+
)
|
|
1203
|
+
);
|
|
1204
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1205
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1206
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1207
|
+
await firstWriter.commit('1/1');
|
|
1208
|
+
|
|
1209
|
+
const second = await factory.updateSyncRules(
|
|
1210
|
+
updateSyncRulesFromYaml(
|
|
1211
|
+
`
|
|
1212
|
+
config:
|
|
1213
|
+
edition: 3
|
|
1214
|
+
|
|
1215
|
+
streams:
|
|
1216
|
+
by_owner:
|
|
1217
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1218
|
+
by_project:
|
|
1219
|
+
query: SELECT * FROM projects WHERE id = subscription.parameter('project_id')
|
|
1220
|
+
`,
|
|
1221
|
+
{ storageVersion }
|
|
1222
|
+
)
|
|
1223
|
+
);
|
|
1224
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1225
|
+
|
|
1226
|
+
const bucketStorage = factory.getInstance(second);
|
|
1227
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1228
|
+
const resolved = await writer.resolveTables({
|
|
1229
|
+
connection_id: 1,
|
|
1230
|
+
source: sourceDescriptor('projects', { objectId: 'projects-relation' }),
|
|
1231
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
|
|
1232
|
+
});
|
|
1233
|
+
expect(resolved.tables).toHaveLength(1);
|
|
1234
|
+
|
|
1235
|
+
await writer.markAllSnapshotDone('2/1');
|
|
1236
|
+
await writer.markTableSnapshotRequired(resolved.tables[0]);
|
|
1237
|
+
|
|
1238
|
+
const configsAfterRequired = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1239
|
+
const statusesAfterRequired = await Promise.all(configsAfterRequired.map((config) => config.getSyncConfigStatus()));
|
|
1240
|
+
const ownerConfigAfterRequired = statusesAfterRequired.find(
|
|
1241
|
+
(status) => status?.state == storage.SyncRuleState.ACTIVE
|
|
1242
|
+
);
|
|
1243
|
+
const projectConfigAfterRequired = statusesAfterRequired.find(
|
|
1244
|
+
(status) => status?.state == storage.SyncRuleState.PROCESSING
|
|
1245
|
+
);
|
|
1246
|
+
expect(ownerConfigAfterRequired?.snapshot_done).toBe(true);
|
|
1247
|
+
expect(projectConfigAfterRequired?.snapshot_done).toBe(false);
|
|
1248
|
+
|
|
1249
|
+
await writer.markTableSnapshotDone(resolved.tables, '3/1');
|
|
1250
|
+
|
|
1251
|
+
const configsAfterDone = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1252
|
+
const statusesAfterDone = await Promise.all(configsAfterDone.map((config) => config.getSyncConfigStatus()));
|
|
1253
|
+
const ownerConfigAfterDone = statusesAfterDone.find((status) => status?.id == ownerConfigAfterRequired!.id);
|
|
1254
|
+
const projectConfigAfterDone = statusesAfterDone.find((status) => status?.id == projectConfigAfterRequired!.id);
|
|
1255
|
+
expect(ownerConfigAfterDone?.snapshot_done).toBe(true);
|
|
1256
|
+
expect(projectConfigAfterDone?.snapshot_done).toBe(false);
|
|
1257
|
+
});
|
|
1258
|
+
|
|
1259
|
+
test.runIf(storageVersion >= 3)(
|
|
1260
|
+
'keeps compatible active and deploying sync configs in one replication stream',
|
|
1261
|
+
async () => {
|
|
1262
|
+
await using factory = await storageConfig.factory();
|
|
1263
|
+
|
|
1264
|
+
const first = await factory.updateSyncRules(
|
|
1265
|
+
updateSyncRulesFromYaml(
|
|
1266
|
+
`
|
|
1267
|
+
config:
|
|
1268
|
+
edition: 3
|
|
1269
|
+
|
|
1270
|
+
streams:
|
|
1271
|
+
by_owner:
|
|
1272
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1273
|
+
`,
|
|
1274
|
+
{ storageVersion }
|
|
1275
|
+
)
|
|
1276
|
+
);
|
|
1277
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1278
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1279
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1280
|
+
await firstWriter.commit('1/1');
|
|
1281
|
+
const initialActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1282
|
+
|
|
1283
|
+
let configs = first.syncConfigContent;
|
|
1284
|
+
const firstConfigId = configs[0].syncConfigId;
|
|
1285
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
|
|
1286
|
+
|
|
1287
|
+
const second = await factory.updateSyncRules(
|
|
1288
|
+
updateSyncRulesFromYaml(
|
|
1289
|
+
`
|
|
1290
|
+
config:
|
|
1291
|
+
edition: 3
|
|
1292
|
+
|
|
1293
|
+
streams:
|
|
1294
|
+
by_project:
|
|
1295
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1296
|
+
`,
|
|
1297
|
+
{ storageVersion }
|
|
1298
|
+
)
|
|
1299
|
+
);
|
|
1300
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1301
|
+
|
|
1302
|
+
configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1303
|
+
const deploying = await factory.getDeployingSyncConfig();
|
|
1304
|
+
expect(configs).toHaveLength(2);
|
|
1305
|
+
expect(deploying).not.toBeNull();
|
|
1306
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
|
|
1307
|
+
|
|
1308
|
+
const stream = (await factory.getActiveSyncConfig())?.replicationStream;
|
|
1309
|
+
expect(stream?.state).toBe(storage.SyncRuleState.ACTIVE);
|
|
1310
|
+
expect(
|
|
1311
|
+
(await Promise.all(configs.map((config) => config.getSyncConfigStatus()))).map((status) => status?.state).sort()
|
|
1312
|
+
).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING]);
|
|
1313
|
+
|
|
1314
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1315
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1316
|
+
for (const config of configs) {
|
|
1317
|
+
expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
|
|
1321
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1322
|
+
await secondWriter.markAllSnapshotDone('2/1');
|
|
1323
|
+
await secondWriter.commit('2/1');
|
|
1324
|
+
|
|
1325
|
+
const updatedStream = (await factory.getActiveSyncConfig())?.replicationStream;
|
|
1326
|
+
expect(updatedStream?.state).toBe(storage.SyncRuleState.ACTIVE);
|
|
1327
|
+
const updatedConfigs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1328
|
+
expect(
|
|
1329
|
+
(await Promise.all(updatedConfigs.map((config) => config.getSyncConfigStatus())))
|
|
1330
|
+
.map((status) => status?.state)
|
|
1331
|
+
.sort()
|
|
1332
|
+
).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.STOP]);
|
|
1333
|
+
expect(await factory.getDeployingSyncConfig()).toBeNull();
|
|
1334
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).not.toBe(firstConfigId);
|
|
1335
|
+
const updatedActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1336
|
+
expect(updatedActiveStorage.replicationStream.replicationJobId).not.toBe(
|
|
1337
|
+
initialActiveStorage.replicationStream.replicationJobId
|
|
1338
|
+
);
|
|
1339
|
+
expect(updatedActiveStorage.replicationStream.replicationJobId).toContain(
|
|
1340
|
+
(await factory.getActiveSyncConfig())!.content.syncConfigId
|
|
1341
|
+
);
|
|
1342
|
+
}
|
|
1343
|
+
);
|
|
1344
|
+
|
|
1345
|
+
test.runIf(storageVersion >= 3)(
|
|
1346
|
+
'restart creates a replacement stream instead of appending to the active stream',
|
|
1347
|
+
async () => {
|
|
1348
|
+
await using factory = await storageConfig.factory();
|
|
1349
|
+
|
|
1350
|
+
const active = await factory.updateSyncRules(
|
|
1351
|
+
updateSyncRulesFromYaml(
|
|
1352
|
+
`
|
|
1353
|
+
config:
|
|
1354
|
+
edition: 3
|
|
1355
|
+
|
|
1356
|
+
streams:
|
|
1357
|
+
by_owner:
|
|
1358
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1359
|
+
`,
|
|
1360
|
+
{ storageVersion }
|
|
1361
|
+
)
|
|
1362
|
+
);
|
|
1363
|
+
const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage;
|
|
1364
|
+
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1365
|
+
await activeWriter.markAllSnapshotDone('1/1');
|
|
1366
|
+
await activeWriter.commit('1/1');
|
|
1367
|
+
|
|
1368
|
+
await factory.restartReplication(active.replicationStreamId);
|
|
1369
|
+
|
|
1370
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1371
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1372
|
+
expect(replicatingStreams[0].replicationStreamId).not.toEqual(active.replicationStreamId);
|
|
1373
|
+
expect(replicatingStreams[0].state).toBe(storage.SyncRuleState.PROCESSING);
|
|
1374
|
+
|
|
1375
|
+
const oldStream = (await (factory as MongoBucketStorage).db.sync_rules.findOne({
|
|
1376
|
+
_id: active.replicationStreamId
|
|
1377
|
+
})) as ReplicationStreamDocumentV3;
|
|
1378
|
+
expect(oldStream.state).toBe(storage.SyncRuleState.ERRORED);
|
|
1379
|
+
expect(oldStream.sync_configs.map((config) => config.state)).toEqual([storage.SyncRuleState.ERRORED]);
|
|
1380
|
+
}
|
|
1381
|
+
);
|
|
1382
|
+
|
|
1383
|
+
test.runIf(storageVersion >= 3)('restart errors an active stream with an embedded deploying config', async () => {
|
|
1384
|
+
await using factory = await storageConfig.factory();
|
|
1385
|
+
|
|
1386
|
+
const active = await factory.updateSyncRules(
|
|
1387
|
+
updateSyncRulesFromYaml(
|
|
1388
|
+
`
|
|
1389
|
+
config:
|
|
1390
|
+
edition: 3
|
|
1391
|
+
|
|
1392
|
+
streams:
|
|
1393
|
+
by_owner:
|
|
1394
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1395
|
+
`,
|
|
1396
|
+
{ storageVersion }
|
|
1397
|
+
)
|
|
1398
|
+
);
|
|
1399
|
+
const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage;
|
|
1400
|
+
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1401
|
+
await activeWriter.markAllSnapshotDone('1/1');
|
|
1402
|
+
await activeWriter.commit('1/1');
|
|
1403
|
+
|
|
1404
|
+
const deploying = await factory.updateSyncRules(
|
|
1405
|
+
updateSyncRulesFromYaml(
|
|
1406
|
+
`
|
|
1407
|
+
config:
|
|
1408
|
+
edition: 3
|
|
1409
|
+
|
|
1410
|
+
streams:
|
|
1411
|
+
by_project:
|
|
1412
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1413
|
+
`,
|
|
1414
|
+
{ storageVersion }
|
|
1415
|
+
)
|
|
1416
|
+
);
|
|
1417
|
+
expect(deploying.replicationStreamId).toEqual(active.replicationStreamId);
|
|
1418
|
+
|
|
1419
|
+
await factory.restartReplication(active.replicationStreamId);
|
|
1420
|
+
|
|
1421
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1422
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1423
|
+
expect(replicatingStreams[0].replicationStreamId).not.toEqual(active.replicationStreamId);
|
|
1424
|
+
|
|
1425
|
+
const oldStream = (await (factory as MongoBucketStorage).db.sync_rules.findOne({
|
|
1426
|
+
_id: active.replicationStreamId
|
|
1427
|
+
})) as ReplicationStreamDocumentV3;
|
|
1428
|
+
expect(oldStream.state).toBe(storage.SyncRuleState.ERRORED);
|
|
1429
|
+
expect(oldStream.sync_configs.map((config) => config.state)).toEqual([
|
|
1430
|
+
storage.SyncRuleState.ERRORED,
|
|
1431
|
+
storage.SyncRuleState.STOP
|
|
1432
|
+
]);
|
|
1433
|
+
const oldActive = await factory.getActiveSyncConfig();
|
|
1434
|
+
expect(oldActive).not.toBeNull();
|
|
1435
|
+
expect(oldActive!.replicationStream.syncConfigContent).toHaveLength(1);
|
|
1436
|
+
await expect(oldActive!.storage.getCheckpoint()).resolves.toBeDefined();
|
|
1437
|
+
});
|
|
1438
|
+
|
|
1439
|
+
test.runIf(storageVersion >= 3)(
|
|
1440
|
+
'appended compatible config adopts the stream checkpoint instead of regressing to 0',
|
|
1441
|
+
async () => {
|
|
1442
|
+
await using factory = await storageConfig.factory();
|
|
1443
|
+
|
|
1444
|
+
// First config replicates some data, advancing the stream-level op head and its checkpoint.
|
|
1445
|
+
const first = await factory.updateSyncRules(
|
|
1446
|
+
updateSyncRulesFromYaml(
|
|
1447
|
+
`
|
|
1448
|
+
config:
|
|
1449
|
+
edition: 3
|
|
1450
|
+
|
|
1451
|
+
streams:
|
|
1452
|
+
by_owner:
|
|
1453
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1454
|
+
`,
|
|
1455
|
+
{ storageVersion }
|
|
1456
|
+
)
|
|
1457
|
+
);
|
|
1458
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1459
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1460
|
+
const sourceTable = await test_utils.resolveTestTable(
|
|
1461
|
+
firstWriter,
|
|
1462
|
+
'todos',
|
|
1463
|
+
['id'],
|
|
1464
|
+
INITIALIZED_MONGO_STORAGE_FACTORY
|
|
1465
|
+
);
|
|
1466
|
+
await firstWriter.save({
|
|
1467
|
+
sourceTable,
|
|
1468
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1469
|
+
after: {
|
|
1470
|
+
id: 'todo-1',
|
|
1471
|
+
owner_id: 'user-1',
|
|
1472
|
+
project_id: 'project-1'
|
|
1473
|
+
},
|
|
1474
|
+
afterReplicaId: test_utils.rid('todo-1')
|
|
1475
|
+
});
|
|
1476
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1477
|
+
await firstWriter.commit('1/1');
|
|
1478
|
+
|
|
1479
|
+
const firstCheckpoint = (await firstStorage.getCheckpoint()).checkpoint;
|
|
1480
|
+
expect(firstCheckpoint).toBeGreaterThan(0n);
|
|
1481
|
+
|
|
1482
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1483
|
+
const streamDocBefore = (await mongoFactory.db.sync_rules.findOne({
|
|
1484
|
+
_id: first.replicationStreamId
|
|
1485
|
+
})) as ReplicationStreamDocumentV3;
|
|
1486
|
+
// The stream-level head was advanced durably to (at least) the first config's checkpoint.
|
|
1487
|
+
expect(streamDocBefore.last_persisted_op).not.toBeNull();
|
|
1488
|
+
expect(BigInt(streamDocBefore.last_persisted_op!)).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1489
|
+
|
|
1490
|
+
// Append a compatible second config that replicates nothing new.
|
|
1491
|
+
const second = await factory.updateSyncRules(
|
|
1492
|
+
updateSyncRulesFromYaml(
|
|
1493
|
+
`
|
|
1494
|
+
config:
|
|
1495
|
+
edition: 3
|
|
1496
|
+
|
|
1497
|
+
streams:
|
|
1498
|
+
by_project:
|
|
1499
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1500
|
+
`,
|
|
1501
|
+
{ storageVersion }
|
|
1502
|
+
)
|
|
1503
|
+
);
|
|
1504
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1505
|
+
|
|
1506
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1507
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1508
|
+
const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
|
|
1509
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1510
|
+
// No new data replicated - just complete the snapshot and commit.
|
|
1511
|
+
await secondWriter.markAllSnapshotDone('2/1');
|
|
1512
|
+
await secondWriter.commit('2/1');
|
|
1513
|
+
|
|
1514
|
+
// The appended config's checkpoint must adopt the stream head, not regress to 0.
|
|
1515
|
+
const streamDocAfter = (await mongoFactory.db.sync_rules.findOne({
|
|
1516
|
+
_id: first.replicationStreamId
|
|
1517
|
+
})) as ReplicationStreamDocumentV3;
|
|
1518
|
+
const head = BigInt(streamDocAfter.last_persisted_op!);
|
|
1519
|
+
for (const config of streamDocAfter.sync_configs) {
|
|
1520
|
+
expect(config.last_checkpoint).not.toBeNull();
|
|
1521
|
+
expect(BigInt(config.last_checkpoint!)).toEqual(head);
|
|
1522
|
+
}
|
|
1523
|
+
// All configs share the same checkpoint, equal to the stream head.
|
|
1524
|
+
expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1525
|
+
|
|
1526
|
+
// After activation, the active config's checkpoint does not regress.
|
|
1527
|
+
const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1528
|
+
const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
|
|
1529
|
+
expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1530
|
+
}
|
|
1531
|
+
);
|
|
1532
|
+
|
|
1533
|
+
test.runIf(storageVersion >= 3)('creates a new replication stream when compatibility options differ', async () => {
|
|
1534
|
+
await using factory = await storageConfig.factory();
|
|
1535
|
+
|
|
1536
|
+
const first = await factory.updateSyncRules(
|
|
1537
|
+
updateSyncRulesFromYaml(
|
|
1538
|
+
`
|
|
1539
|
+
config:
|
|
1540
|
+
edition: 3
|
|
1541
|
+
|
|
1542
|
+
streams:
|
|
1543
|
+
by_owner:
|
|
1544
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1545
|
+
`,
|
|
1546
|
+
{ storageVersion }
|
|
1547
|
+
)
|
|
1548
|
+
);
|
|
1549
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1550
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1551
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1552
|
+
await firstWriter.commit('1/1');
|
|
1553
|
+
|
|
1554
|
+
// Same streams, but a different compatibility edition - must not append to the active stream.
|
|
1555
|
+
const second = await factory.updateSyncRules(
|
|
1556
|
+
updateSyncRulesFromYaml(
|
|
1557
|
+
`
|
|
1558
|
+
config:
|
|
1559
|
+
edition: 2
|
|
1560
|
+
|
|
1561
|
+
streams:
|
|
1562
|
+
by_owner:
|
|
1563
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1564
|
+
`,
|
|
1565
|
+
{ storageVersion }
|
|
1566
|
+
)
|
|
1567
|
+
);
|
|
1568
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1569
|
+
|
|
1570
|
+
// The first stream stays active until the replacement has replicated.
|
|
1571
|
+
const active = await factory.getActiveSyncConfig();
|
|
1572
|
+
expect(active?.replicationStream.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1573
|
+
expect((await factory.getDeployingSyncConfig())?.replicationStream.replicationStreamId).toEqual(
|
|
1574
|
+
second.replicationStreamId
|
|
1575
|
+
);
|
|
1576
|
+
});
|
|
1577
|
+
|
|
1578
|
+
test.runIf(storageVersion >= 3)(
|
|
1579
|
+
'creates a new replication stream for legacy sync rules without a serialized plan',
|
|
1580
|
+
async () => {
|
|
1581
|
+
await using factory = await storageConfig.factory();
|
|
1582
|
+
|
|
1583
|
+
const first = await factory.updateSyncRules(
|
|
1584
|
+
updateSyncRulesFromYaml(
|
|
1585
|
+
`
|
|
1586
|
+
config:
|
|
1587
|
+
edition: 3
|
|
1588
|
+
|
|
1589
|
+
streams:
|
|
1590
|
+
by_owner:
|
|
1591
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1592
|
+
`,
|
|
1593
|
+
{ storageVersion }
|
|
1594
|
+
)
|
|
1595
|
+
);
|
|
1596
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1597
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1598
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1599
|
+
await firstWriter.commit('1/1');
|
|
1600
|
+
|
|
1601
|
+
// Legacy sync rules have no serialized plan - must not append to the active stream.
|
|
1602
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1603
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1604
|
+
}
|
|
1605
|
+
);
|
|
1606
|
+
|
|
1607
|
+
test.runIf(storageVersion >= 3)('does not append to an active legacy sync config', async () => {
|
|
1608
|
+
await using factory = await storageConfig.factory();
|
|
1609
|
+
|
|
1610
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1611
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1612
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1613
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1614
|
+
await firstWriter.commit('1/1');
|
|
1615
|
+
|
|
1616
|
+
// The active config has no serialized plan, so a new streams config must not be appended to it.
|
|
1617
|
+
const second = await factory.updateSyncRules(
|
|
1618
|
+
updateSyncRulesFromYaml(
|
|
1619
|
+
`
|
|
1620
|
+
config:
|
|
1621
|
+
edition: 3
|
|
1622
|
+
|
|
1623
|
+
streams:
|
|
1624
|
+
by_owner:
|
|
1625
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1626
|
+
`,
|
|
1627
|
+
{ storageVersion }
|
|
1628
|
+
)
|
|
1629
|
+
);
|
|
1630
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1631
|
+
});
|
|
1632
|
+
|
|
591
1633
|
test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
|
|
592
1634
|
await using factory = await storageConfig.factory();
|
|
593
1635
|
|
|
@@ -600,7 +1642,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
600
1642
|
).resolves.toBeDefined();
|
|
601
1643
|
|
|
602
1644
|
const mongoFactory = factory as MongoBucketStorage;
|
|
603
|
-
expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.
|
|
1645
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.replicationStreamId }))?.state).toBe(
|
|
604
1646
|
storage.SyncRuleState.STOP
|
|
605
1647
|
);
|
|
606
1648
|
});
|
|
@@ -612,10 +1654,48 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
612
1654
|
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
|
|
613
1655
|
);
|
|
614
1656
|
|
|
615
|
-
expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.
|
|
1657
|
+
expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.replicationStreamId);
|
|
616
1658
|
await syncRules.current_lock?.release();
|
|
617
1659
|
});
|
|
618
1660
|
|
|
1661
|
+
test.runIf(storageVersion >= 3)('does not lock when appending a sync config to an existing stream', async () => {
|
|
1662
|
+
await using factory = await storageConfig.factory();
|
|
1663
|
+
|
|
1664
|
+
const firstRules = `
|
|
1665
|
+
config:
|
|
1666
|
+
edition: 3
|
|
1667
|
+
|
|
1668
|
+
streams:
|
|
1669
|
+
by_owner:
|
|
1670
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1671
|
+
`;
|
|
1672
|
+
const secondRules = `
|
|
1673
|
+
config:
|
|
1674
|
+
edition: 3
|
|
1675
|
+
|
|
1676
|
+
streams:
|
|
1677
|
+
by_project:
|
|
1678
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1679
|
+
`;
|
|
1680
|
+
|
|
1681
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstRules, { storageVersion, lock: true }));
|
|
1682
|
+
expect(first.current_lock?.sync_rules_id).toBe(first.replicationStreamId);
|
|
1683
|
+
try {
|
|
1684
|
+
const firstStorage = factory.getInstance(first);
|
|
1685
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1686
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1687
|
+
await firstWriter.commit('1/1');
|
|
1688
|
+
|
|
1689
|
+
const second = await factory.updateSyncRules(
|
|
1690
|
+
updateSyncRulesFromYaml(secondRules, { storageVersion, lock: true })
|
|
1691
|
+
);
|
|
1692
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1693
|
+
expect(second.current_lock).toBeNull();
|
|
1694
|
+
} finally {
|
|
1695
|
+
await first.current_lock?.release();
|
|
1696
|
+
}
|
|
1697
|
+
});
|
|
1698
|
+
|
|
619
1699
|
test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
|
|
620
1700
|
await using factory = await storageConfig.factory();
|
|
621
1701
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -647,10 +1727,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
647
1727
|
await writer.commit('1/1');
|
|
648
1728
|
|
|
649
1729
|
const mongoFactory = factory as MongoBucketStorage;
|
|
650
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1730
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
651
1731
|
|
|
652
1732
|
const sourceRecordCollections = await mongoFactory.db.db
|
|
653
|
-
.listCollections({ name: new RegExp(`^source_records_${syncRules.
|
|
1733
|
+
.listCollections({ name: new RegExp(`^source_records_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
|
|
654
1734
|
.toArray();
|
|
655
1735
|
expect(sourceRecordCollections).toEqual([]);
|
|
656
1736
|
});
|
|
@@ -686,13 +1766,103 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
686
1766
|
await writer.commit('1/1');
|
|
687
1767
|
|
|
688
1768
|
const mongoFactory = factory as MongoBucketStorage;
|
|
689
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1769
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
690
1770
|
|
|
691
1771
|
await bucketStorage.clear();
|
|
692
1772
|
|
|
693
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1773
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
|
|
694
1774
|
});
|
|
695
1775
|
|
|
1776
|
+
test.runIf(storageVersion < 3)(
|
|
1777
|
+
'clear removes large v1 current_data sets without affecting other streams',
|
|
1778
|
+
async () => {
|
|
1779
|
+
await using factory = await storageConfig.factory();
|
|
1780
|
+
const syncRules = await factory.updateSyncRules(
|
|
1781
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
|
|
1782
|
+
storageVersion
|
|
1783
|
+
})
|
|
1784
|
+
);
|
|
1785
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1786
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1787
|
+
const sourceTableId = new bson.ObjectId();
|
|
1788
|
+
const documents = Array.from({ length: 10_002 }, (_, index) => ({
|
|
1789
|
+
_id: {
|
|
1790
|
+
g: syncRules.replicationStreamId,
|
|
1791
|
+
t: sourceTableId,
|
|
1792
|
+
k: index
|
|
1793
|
+
},
|
|
1794
|
+
data: new bson.Binary(),
|
|
1795
|
+
buckets: [],
|
|
1796
|
+
lookups: []
|
|
1797
|
+
}));
|
|
1798
|
+
await mongoFactory.db.current_data.insertMany([
|
|
1799
|
+
...documents,
|
|
1800
|
+
{
|
|
1801
|
+
_id: {
|
|
1802
|
+
g: syncRules.replicationStreamId + 1,
|
|
1803
|
+
t: sourceTableId,
|
|
1804
|
+
k: 0
|
|
1805
|
+
},
|
|
1806
|
+
data: new bson.Binary(),
|
|
1807
|
+
buckets: [],
|
|
1808
|
+
lookups: []
|
|
1809
|
+
}
|
|
1810
|
+
]);
|
|
1811
|
+
|
|
1812
|
+
await bucketStorage.clear();
|
|
1813
|
+
|
|
1814
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
|
|
1815
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId + 1 })).toBe(1);
|
|
1816
|
+
}
|
|
1817
|
+
);
|
|
1818
|
+
|
|
1819
|
+
test.runIf(storageVersion < 3)(
|
|
1820
|
+
'clear removes large v1 parameter index sets without affecting other streams',
|
|
1821
|
+
async () => {
|
|
1822
|
+
await using factory = await storageConfig.factory();
|
|
1823
|
+
const syncRules = await factory.updateSyncRules(
|
|
1824
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
|
|
1825
|
+
storageVersion
|
|
1826
|
+
})
|
|
1827
|
+
);
|
|
1828
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1829
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1830
|
+
const sourceTableId = new bson.ObjectId();
|
|
1831
|
+
const documents = Array.from({ length: 10_002 }, (_, index) => ({
|
|
1832
|
+
_id: BigInt(index * 2),
|
|
1833
|
+
key: {
|
|
1834
|
+
g: syncRules.replicationStreamId,
|
|
1835
|
+
t: sourceTableId,
|
|
1836
|
+
k: index
|
|
1837
|
+
},
|
|
1838
|
+
lookup: new bson.Binary(),
|
|
1839
|
+
bucket_parameters: []
|
|
1840
|
+
}));
|
|
1841
|
+
await mongoFactory.db.bucket_parameters.insertMany([
|
|
1842
|
+
...documents,
|
|
1843
|
+
{
|
|
1844
|
+
_id: 1n,
|
|
1845
|
+
key: {
|
|
1846
|
+
g: syncRules.replicationStreamId + 1,
|
|
1847
|
+
t: sourceTableId,
|
|
1848
|
+
k: 0
|
|
1849
|
+
},
|
|
1850
|
+
lookup: new bson.Binary(),
|
|
1851
|
+
bucket_parameters: []
|
|
1852
|
+
}
|
|
1853
|
+
]);
|
|
1854
|
+
|
|
1855
|
+
await bucketStorage.clear();
|
|
1856
|
+
|
|
1857
|
+
expect(await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId })).toBe(
|
|
1858
|
+
0
|
|
1859
|
+
);
|
|
1860
|
+
expect(
|
|
1861
|
+
await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId + 1 })
|
|
1862
|
+
).toBe(1);
|
|
1863
|
+
}
|
|
1864
|
+
);
|
|
1865
|
+
|
|
696
1866
|
test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
|
|
697
1867
|
await using factory = await storageConfig.factory();
|
|
698
1868
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -725,12 +1895,28 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
725
1895
|
await writer.commit('1/1');
|
|
726
1896
|
|
|
727
1897
|
const mongoFactory = factory as MongoBucketStorage;
|
|
728
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1898
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
729
1899
|
|
|
730
1900
|
const metricsAfter = await factory.getStorageMetrics();
|
|
731
1901
|
expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
|
|
732
1902
|
});
|
|
733
1903
|
|
|
1904
|
+
test.runIf(storageVersion >= 3)('storage metrics include the sync config version label', async () => {
|
|
1905
|
+
await using factory = await storageConfig.factory();
|
|
1906
|
+
const syncRules = await factory.updateSyncRules(
|
|
1907
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, version_label: 'metrics-test' })
|
|
1908
|
+
);
|
|
1909
|
+
|
|
1910
|
+
const metrics = await factory.getStorageMetrics();
|
|
1911
|
+
|
|
1912
|
+
expect(metrics.sync_config_metrics).toContainEqual(
|
|
1913
|
+
expect.objectContaining({
|
|
1914
|
+
sync_config_id: syncRules.syncConfigContent[0].syncConfigId,
|
|
1915
|
+
version_label: 'metrics-test'
|
|
1916
|
+
})
|
|
1917
|
+
);
|
|
1918
|
+
});
|
|
1919
|
+
|
|
734
1920
|
test.runIf(storageVersion >= 3)(
|
|
735
1921
|
'loads parameter checkpoint changes across all v3 parameter index collections',
|
|
736
1922
|
async () => {
|
|
@@ -798,25 +1984,25 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
798
1984
|
);
|
|
799
1985
|
|
|
800
1986
|
const mongoFactory = factory as MongoBucketStorage;
|
|
801
|
-
const bucketStorage = mongoFactory.getInstance(syncRules)
|
|
802
|
-
const db = bucketStorage.db;
|
|
803
|
-
await db.initializeStreamStorage(syncRules.
|
|
1987
|
+
const bucketStorage = mongoFactory.getInstance(syncRules);
|
|
1988
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1989
|
+
await db.initializeStreamStorage(syncRules.replicationStreamId);
|
|
804
1990
|
|
|
805
1991
|
const sourceTableA = new bson.ObjectId();
|
|
806
1992
|
const sourceTableB = new bson.ObjectId();
|
|
807
|
-
await db.
|
|
1993
|
+
await db.sourceTables(syncRules.replicationStreamId).insertMany([
|
|
808
1994
|
{
|
|
809
1995
|
_id: sourceTableA,
|
|
810
1996
|
connection_id: 1,
|
|
811
1997
|
relation_id: 'a',
|
|
812
1998
|
schema_name: 'public',
|
|
813
1999
|
table_name: 'table_a',
|
|
814
|
-
replica_id_columns:
|
|
815
|
-
replica_id_columns2: [],
|
|
2000
|
+
replica_id_columns: [],
|
|
816
2001
|
snapshot_done: true,
|
|
817
2002
|
snapshot_status: undefined,
|
|
818
2003
|
bucket_data_source_ids: [],
|
|
819
2004
|
parameter_lookup_source_ids: [],
|
|
2005
|
+
event_definition_ids: [],
|
|
820
2006
|
latest_pending_delete: 9n
|
|
821
2007
|
},
|
|
822
2008
|
{
|
|
@@ -825,45 +2011,63 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
825
2011
|
relation_id: 'b',
|
|
826
2012
|
schema_name: 'public',
|
|
827
2013
|
table_name: 'table_b',
|
|
828
|
-
replica_id_columns:
|
|
829
|
-
replica_id_columns2: [],
|
|
2014
|
+
replica_id_columns: [],
|
|
830
2015
|
snapshot_done: true,
|
|
831
2016
|
snapshot_status: undefined,
|
|
832
2017
|
bucket_data_source_ids: [],
|
|
833
2018
|
parameter_lookup_source_ids: [],
|
|
2019
|
+
event_definition_ids: [],
|
|
834
2020
|
latest_pending_delete: 12n
|
|
835
2021
|
}
|
|
836
2022
|
]);
|
|
837
2023
|
|
|
838
|
-
await db.
|
|
2024
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).insertMany([
|
|
839
2025
|
{ _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
|
|
840
2026
|
{ _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
|
|
841
2027
|
{ _id: 'active', data: null, buckets: [], lookups: [] }
|
|
842
2028
|
]);
|
|
843
2029
|
await db
|
|
844
|
-
.
|
|
2030
|
+
.sourceRecords(syncRules.replicationStreamId, sourceTableB)
|
|
845
2031
|
.insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
|
|
846
2032
|
|
|
847
|
-
const store = new SourceRecordStoreV3(
|
|
2033
|
+
const store = new SourceRecordStoreV3(
|
|
2034
|
+
db,
|
|
2035
|
+
syncRules.replicationStreamId,
|
|
2036
|
+
bucketStorage.replicationStream.syncConfigContent[0].mapping
|
|
2037
|
+
);
|
|
848
2038
|
const logger = { info() {} } as any;
|
|
849
2039
|
|
|
850
2040
|
await store.postCommitCleanup(6n, logger);
|
|
851
2041
|
|
|
852
|
-
expect(
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
expect(
|
|
856
|
-
|
|
2042
|
+
expect(
|
|
2043
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 5n })
|
|
2044
|
+
).toBe(0);
|
|
2045
|
+
expect(
|
|
2046
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 9n })
|
|
2047
|
+
).toBe(1);
|
|
2048
|
+
expect(
|
|
2049
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableB).countDocuments({ pending_delete: 12n })
|
|
2050
|
+
).toBe(1);
|
|
2051
|
+
expect(
|
|
2052
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
|
|
2053
|
+
).toBe(9n);
|
|
2054
|
+
expect(
|
|
2055
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
|
|
2056
|
+
).toBe(12n);
|
|
857
2057
|
|
|
858
2058
|
await store.postCommitCleanup(10n, logger);
|
|
859
2059
|
|
|
860
2060
|
expect(
|
|
861
|
-
await db
|
|
2061
|
+
await db
|
|
2062
|
+
.sourceRecords(syncRules.replicationStreamId, sourceTableA)
|
|
2063
|
+
.countDocuments({ pending_delete: { $exists: true } })
|
|
862
2064
|
).toBe(0);
|
|
863
2065
|
expect(
|
|
864
|
-
(await db.
|
|
2066
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
|
|
865
2067
|
).toBeUndefined();
|
|
866
|
-
expect(
|
|
2068
|
+
expect(
|
|
2069
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
|
|
2070
|
+
).toBe(12n);
|
|
867
2071
|
});
|
|
868
2072
|
}
|
|
869
2073
|
|
|
@@ -880,7 +2084,7 @@ describe('sync - mongodb', () => {
|
|
|
880
2084
|
await legacyWriter.markAllSnapshotDone('1/1');
|
|
881
2085
|
await legacyWriter.commit('1/1');
|
|
882
2086
|
|
|
883
|
-
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.
|
|
2087
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
|
|
884
2088
|
storage.SyncRuleState.ACTIVE
|
|
885
2089
|
);
|
|
886
2090
|
|
|
@@ -892,7 +2096,7 @@ describe('sync - mongodb', () => {
|
|
|
892
2096
|
await v3Writer.markAllSnapshotDone('2/1');
|
|
893
2097
|
await v3Writer.commit('2/1');
|
|
894
2098
|
|
|
895
|
-
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.
|
|
2099
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
|
|
896
2100
|
storage.SyncRuleState.STOP
|
|
897
2101
|
);
|
|
898
2102
|
});
|
|
@@ -900,6 +2104,233 @@ describe('sync - mongodb', () => {
|
|
|
900
2104
|
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
901
2105
|
describe(`storage v${storageVersion}`, () => {
|
|
902
2106
|
registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
|
|
2107
|
+
|
|
2108
|
+
describe.runIf(storageVersion >= 3)('V3 read filtering boundaries', () => {
|
|
2109
|
+
async function setupFilteringTest() {
|
|
2110
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
2111
|
+
const syncRules = await factory.updateSyncRules(
|
|
2112
|
+
updateSyncRulesFromYaml(
|
|
2113
|
+
`
|
|
2114
|
+
bucket_definitions:
|
|
2115
|
+
global:
|
|
2116
|
+
data:
|
|
2117
|
+
- SELECT id, description FROM test
|
|
2118
|
+
`,
|
|
2119
|
+
{ storageVersion }
|
|
2120
|
+
)
|
|
2121
|
+
);
|
|
2122
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
2123
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
2124
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
2125
|
+
|
|
2126
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
2127
|
+
const definitionId = mapping.bucketSourceId(request.source);
|
|
2128
|
+
const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
|
|
2129
|
+
|
|
2130
|
+
const bucketName = request.bucket;
|
|
2131
|
+
const sourceTable = new bson.ObjectId();
|
|
2132
|
+
const bucketKey: BucketKey = {
|
|
2133
|
+
replicationStreamId: syncRules.replicationStreamId,
|
|
2134
|
+
definitionId,
|
|
2135
|
+
bucket: bucketName
|
|
2136
|
+
};
|
|
2137
|
+
|
|
2138
|
+
function makeOps(opIds: bigint[]): BucketDataDoc[] {
|
|
2139
|
+
return opIds.map((opId) => ({
|
|
2140
|
+
bucketKey,
|
|
2141
|
+
o: opId,
|
|
2142
|
+
op: 'PUT' as const,
|
|
2143
|
+
source_table: sourceTable,
|
|
2144
|
+
source_key: test_utils.rid(`row-${opId}`),
|
|
2145
|
+
table: 'items',
|
|
2146
|
+
row_id: `row-${opId}`,
|
|
2147
|
+
checksum: BigInt(opId) * 10n,
|
|
2148
|
+
data: `{"id":"row-${opId}"}`
|
|
2149
|
+
}));
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
const docA = serializeBucketData(bucketName, makeOps([10n, 20n, 30n]));
|
|
2153
|
+
const docB = serializeBucketData(bucketName, makeOps([40n, 50n, 60n]));
|
|
2154
|
+
const docC = serializeBucketData(bucketName, makeOps([70n, 80n, 90n]));
|
|
2155
|
+
|
|
2156
|
+
await collection.insertMany([docA, docB, docC]);
|
|
2157
|
+
|
|
2158
|
+
return { factory, syncRules, bucketStorage, bucketName };
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
|
|
2162
|
+
const { syncRules, bucketStorage } = await setupFilteringTest();
|
|
2163
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
|
|
2164
|
+
const batch = await test_utils.getBatchArray(
|
|
2165
|
+
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(BigInt(checkpoint)), [request])
|
|
2166
|
+
);
|
|
2167
|
+
const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
|
|
2168
|
+
return ops;
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
test('case 1: start=5, checkpoint=95 → all ops', async () => {
|
|
2172
|
+
const ops = await getFilteredOps(5, 95);
|
|
2173
|
+
expect(ops).toEqual([10n, 20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
|
|
2174
|
+
});
|
|
2175
|
+
|
|
2176
|
+
test('case 2: start=10, checkpoint=90 → ops in (10,90]', async () => {
|
|
2177
|
+
const ops = await getFilteredOps(10, 90);
|
|
2178
|
+
expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
|
|
2179
|
+
});
|
|
2180
|
+
|
|
2181
|
+
test('case 3: start=15, checkpoint=85 → partial doc boundaries', async () => {
|
|
2182
|
+
const ops = await getFilteredOps(15, 85);
|
|
2183
|
+
expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n]);
|
|
2184
|
+
});
|
|
2185
|
+
|
|
2186
|
+
test('case 4: start=25, checkpoint=55 → spans two docs', async () => {
|
|
2187
|
+
const ops = await getFilteredOps(25, 55);
|
|
2188
|
+
expect(ops).toEqual([30n, 40n, 50n]);
|
|
2189
|
+
});
|
|
2190
|
+
|
|
2191
|
+
test('case 5: start=35, checkpoint=45 → single op within doc', async () => {
|
|
2192
|
+
const ops = await getFilteredOps(35, 45);
|
|
2193
|
+
expect(ops).toEqual([40n]);
|
|
2194
|
+
});
|
|
2195
|
+
|
|
2196
|
+
test('case 6: start=35, checkpoint=65 → full doc B', async () => {
|
|
2197
|
+
const ops = await getFilteredOps(35, 65);
|
|
2198
|
+
expect(ops).toEqual([40n, 50n, 60n]);
|
|
2199
|
+
});
|
|
2200
|
+
|
|
2201
|
+
test('case 7: start=25, checkpoint=35 → single op from doc A', async () => {
|
|
2202
|
+
const ops = await getFilteredOps(25, 35);
|
|
2203
|
+
expect(ops).toEqual([30n]);
|
|
2204
|
+
});
|
|
2205
|
+
|
|
2206
|
+
test('case 8: start=30, checkpoint=40 → op at checkpoint from next doc', async () => {
|
|
2207
|
+
const ops = await getFilteredOps(30, 40);
|
|
2208
|
+
expect(ops).toEqual([40n]);
|
|
2209
|
+
});
|
|
2210
|
+
|
|
2211
|
+
test('case 9: start=100, checkpoint=200 → beyond all docs', async () => {
|
|
2212
|
+
const ops = await getFilteredOps(100, 200);
|
|
2213
|
+
expect(ops).toEqual([]);
|
|
2214
|
+
});
|
|
2215
|
+
|
|
2216
|
+
test('case 10: start=0, checkpoint=5 → before all docs', async () => {
|
|
2217
|
+
const ops = await getFilteredOps(0, 5);
|
|
2218
|
+
expect(ops).toEqual([]);
|
|
2219
|
+
});
|
|
2220
|
+
|
|
2221
|
+
test('case 11: start=50, checkpoint=50 → zero-width range', async () => {
|
|
2222
|
+
const ops = await getFilteredOps(50, 50);
|
|
2223
|
+
expect(ops).toEqual([]);
|
|
2224
|
+
});
|
|
2225
|
+
|
|
2226
|
+
test('case 12: start=45, checkpoint=50 → op at checkpoint boundary', async () => {
|
|
2227
|
+
const ops = await getFilteredOps(45, 50);
|
|
2228
|
+
expect(ops).toEqual([50n]);
|
|
2229
|
+
});
|
|
2230
|
+
|
|
2231
|
+
test('case 13: start=50, checkpoint=55 → no ops strictly after start', async () => {
|
|
2232
|
+
const ops = await getFilteredOps(50, 55);
|
|
2233
|
+
expect(ops).toEqual([]);
|
|
2234
|
+
});
|
|
2235
|
+
|
|
2236
|
+
test('all-filtered first batch still returns data behind the batch boundary', async () => {
|
|
2237
|
+
// Documents straddling the requested (start, end] window are matched by the
|
|
2238
|
+
// query, but contribute no rows after filtering. If an entire server batch
|
|
2239
|
+
// (~101 documents) consists of such straddlers, the remaining documents in
|
|
2240
|
+
// the cursor must still be reachable. Storage reports the straddler buckets
|
|
2241
|
+
// as complete via empty chunks, and the caller re-requests the rest.
|
|
2242
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
2243
|
+
const syncRules = await factory.updateSyncRules(
|
|
2244
|
+
updateSyncRulesFromYaml(
|
|
2245
|
+
`
|
|
2246
|
+
bucket_definitions:
|
|
2247
|
+
by_user:
|
|
2248
|
+
parameters: select request.user_id() as user_id
|
|
2249
|
+
data: [select * from test where owner_id = bucket.user_id]
|
|
2250
|
+
`,
|
|
2251
|
+
{ storageVersion }
|
|
2252
|
+
)
|
|
2253
|
+
);
|
|
2254
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3;
|
|
2255
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
2256
|
+
|
|
2257
|
+
const start = 5n;
|
|
2258
|
+
const end = 50n;
|
|
2259
|
+
|
|
2260
|
+
// 150 buckets sorted before the data bucket, each with a single document
|
|
2261
|
+
// containing ops at 1 and 100: matched (_id.o=100 > start, min_op=1 <= end),
|
|
2262
|
+
// but no op in (5, 50].
|
|
2263
|
+
const straddlerNames = Array.from({ length: 150 }, (_, i) => `b${`${i}`.padStart(3, '0')}`);
|
|
2264
|
+
const requests = [...straddlerNames, 'zzz'].map((id) =>
|
|
2265
|
+
bucketRequest(syncRules.syncConfigContent[0], `by_user["${id}"]`, start)
|
|
2266
|
+
);
|
|
2267
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
2268
|
+
const definitionId = mapping.bucketSourceId(requests[0].source);
|
|
2269
|
+
const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
|
|
2270
|
+
const sourceTable = new bson.ObjectId();
|
|
2271
|
+
|
|
2272
|
+
function makeOps(bucket: string, opIds: bigint[]): BucketDataDoc[] {
|
|
2273
|
+
const bucketKey: BucketKey = {
|
|
2274
|
+
replicationStreamId: syncRules.replicationStreamId,
|
|
2275
|
+
definitionId,
|
|
2276
|
+
bucket
|
|
2277
|
+
};
|
|
2278
|
+
return opIds.map((opId) => ({
|
|
2279
|
+
bucketKey,
|
|
2280
|
+
o: opId,
|
|
2281
|
+
op: 'PUT' as const,
|
|
2282
|
+
source_table: sourceTable,
|
|
2283
|
+
source_key: test_utils.rid(`row-${opId}`),
|
|
2284
|
+
table: 'test',
|
|
2285
|
+
row_id: `row-${opId}`,
|
|
2286
|
+
checksum: BigInt(opId) * 10n,
|
|
2287
|
+
data: `{"id":"row-${opId}"}`
|
|
2288
|
+
}));
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2291
|
+
const straddlerDocs = requests
|
|
2292
|
+
.slice(0, -1)
|
|
2293
|
+
.map((request) => serializeBucketData(request.bucket, makeOps(request.bucket, [1n, 100n])));
|
|
2294
|
+
const dataBucket = requests[requests.length - 1].bucket;
|
|
2295
|
+
const dataDoc = serializeBucketData(dataBucket, makeOps(dataBucket, [10n]));
|
|
2296
|
+
await collection.insertMany([...straddlerDocs, dataDoc]);
|
|
2297
|
+
|
|
2298
|
+
// Emulate the caller loop in sync.ts / BucketChecksumState: advance bucket
|
|
2299
|
+
// positions from each chunk, drop completed buckets, and re-request while
|
|
2300
|
+
// any chunk reported has_more.
|
|
2301
|
+
const positions = new Map(requests.map((request) => [request.bucket, request.start]));
|
|
2302
|
+
const pending = new Set(positions.keys());
|
|
2303
|
+
const receivedOps: bigint[] = [];
|
|
2304
|
+
let rounds = 0;
|
|
2305
|
+
|
|
2306
|
+
while (rounds < 10) {
|
|
2307
|
+
rounds++;
|
|
2308
|
+
const roundRequests = requests
|
|
2309
|
+
.filter((request) => pending.has(request.bucket))
|
|
2310
|
+
.map((request) => ({ ...request, start: positions.get(request.bucket)! }));
|
|
2311
|
+
const batch = await test_utils.getBatchArray(
|
|
2312
|
+
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(end), roundRequests)
|
|
2313
|
+
);
|
|
2314
|
+
let anyHasMore = false;
|
|
2315
|
+
for (const { chunkData } of batch) {
|
|
2316
|
+
positions.set(chunkData.bucket, BigInt(chunkData.next_after));
|
|
2317
|
+
if (chunkData.has_more) {
|
|
2318
|
+
anyHasMore = true;
|
|
2319
|
+
} else {
|
|
2320
|
+
pending.delete(chunkData.bucket);
|
|
2321
|
+
}
|
|
2322
|
+
receivedOps.push(...chunkData.data.map((entry) => BigInt(entry.op_id)));
|
|
2323
|
+
}
|
|
2324
|
+
if (!anyHasMore) {
|
|
2325
|
+
break;
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
// The op behind the all-straddler first batch must be returned.
|
|
2330
|
+
expect(receivedOps).toEqual([10n]);
|
|
2331
|
+
expect(rounds).toBeLessThan(10);
|
|
2332
|
+
});
|
|
2333
|
+
});
|
|
903
2334
|
});
|
|
904
2335
|
}
|
|
905
2336
|
});
|