@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,13 +1,24 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
|
|
2
|
+
import {
|
|
3
|
+
deserializeParameterLookup,
|
|
4
|
+
JwtPayload,
|
|
5
|
+
SingleSyncConfigBucketDefinitionMapping,
|
|
6
|
+
storage,
|
|
7
|
+
updateSyncRulesFromYaml
|
|
8
|
+
} from '@powersync/service-core';
|
|
2
9
|
import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
|
|
3
|
-
import {
|
|
10
|
+
import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
4
11
|
import * as bson from 'bson';
|
|
5
|
-
import * as sqlite from 'node:sqlite';
|
|
6
12
|
import { describe, expect, test } from 'vitest';
|
|
7
13
|
import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
|
|
14
|
+
import { MongoParsedSyncConfigSet } from '../../src/storage/implementation/MongoParsedSyncConfigSet.js';
|
|
15
|
+
import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementation/MongoPersistedSyncConfigContent.js';
|
|
16
|
+
import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
|
|
8
17
|
import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
|
|
18
|
+
import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
|
|
9
19
|
import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
|
|
10
20
|
import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
21
|
+
import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
|
|
11
22
|
import {
|
|
12
23
|
CurrentBucketV3,
|
|
13
24
|
ReplicationStreamDocumentV3,
|
|
@@ -53,10 +64,54 @@ function objectIdGenerator(id: string) {
|
|
|
53
64
|
};
|
|
54
65
|
}
|
|
55
66
|
|
|
56
|
-
|
|
67
|
+
/**
|
|
68
|
+
* Build a standalone parsed sync config set for the given yaml, with its own mapping.
|
|
69
|
+
*
|
|
70
|
+
* This keeps the sync rules and the bucket definition mapping paired, as required by
|
|
71
|
+
* `ResolveTablesOptions.parsedSyncConfig`.
|
|
72
|
+
*/
|
|
73
|
+
function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
|
|
57
74
|
const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
|
|
58
75
|
expect(parsed.errors).toEqual([]);
|
|
59
|
-
return
|
|
76
|
+
return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
|
|
77
|
+
{
|
|
78
|
+
syncConfigId: new bson.ObjectId().toHexString(),
|
|
79
|
+
syncConfig: parsed,
|
|
80
|
+
mapping: SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(parsed)
|
|
81
|
+
}
|
|
82
|
+
]);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory, replicationStreamId: number) {
|
|
86
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
87
|
+
const doc = (await mongoFactory.db.sync_rules.findOne({
|
|
88
|
+
_id: replicationStreamId
|
|
89
|
+
})) as ReplicationStreamDocumentV3 | null;
|
|
90
|
+
if (doc == null) {
|
|
91
|
+
return [];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const db = mongoFactory.db.versioned(getMongoStorageConfig(doc.storage_version)) as VersionedPowerSyncMongoV3;
|
|
95
|
+
const syncConfigDocs = await db.syncConfigDefinitions
|
|
96
|
+
.find({
|
|
97
|
+
_id: { $in: doc.sync_configs.map((config) => config._id) }
|
|
98
|
+
})
|
|
99
|
+
.toArray();
|
|
100
|
+
|
|
101
|
+
return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Get a MongoDB V3 storage bucket definition id, from a bucket name.
|
|
106
|
+
*
|
|
107
|
+
* This relies on internals of bucket naming, but helps for simplifying tests.
|
|
108
|
+
*/
|
|
109
|
+
function bucketDefinitionId(bucket: string) {
|
|
110
|
+
const match = bucket.match(/^\w+\|\w+\.\w+\.(\w+)\[/);
|
|
111
|
+
if (match == null) {
|
|
112
|
+
throw new Error(`Expected versioned bucket name, got ${bucket}`);
|
|
113
|
+
}
|
|
114
|
+
return parseInt(match[1], 16);
|
|
60
115
|
}
|
|
61
116
|
|
|
62
117
|
function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
|
|
@@ -82,6 +137,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
82
137
|
)
|
|
83
138
|
);
|
|
84
139
|
const bucketStorage = factory.getInstance(syncRules);
|
|
140
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
85
141
|
|
|
86
142
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
87
143
|
|
|
@@ -135,8 +191,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
135
191
|
const checkpoint = flushResult!.flushed_op;
|
|
136
192
|
|
|
137
193
|
const options: storage.BucketDataBatchOptions = {};
|
|
138
|
-
const batch1 = await test_utils.
|
|
139
|
-
bucketStorage.getBucketDataBatch(
|
|
194
|
+
const batch1 = await test_utils.getBatchArray(
|
|
195
|
+
bucketStorage.getBucketDataBatch(
|
|
196
|
+
test_utils.testCheckpoint(checkpoint),
|
|
197
|
+
[bucketRequest(syncRulesContent, 'global[]', 0n)],
|
|
198
|
+
options
|
|
199
|
+
)
|
|
140
200
|
);
|
|
141
201
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
142
202
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
@@ -148,10 +208,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
148
208
|
next_after: '2'
|
|
149
209
|
});
|
|
150
210
|
|
|
151
|
-
const batch2 = await test_utils.
|
|
211
|
+
const batch2 = await test_utils.getBatchArray(
|
|
152
212
|
bucketStorage.getBucketDataBatch(
|
|
153
|
-
checkpoint,
|
|
154
|
-
[bucketRequest(
|
|
213
|
+
test_utils.testCheckpoint(checkpoint),
|
|
214
|
+
[bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
|
|
155
215
|
options
|
|
156
216
|
)
|
|
157
217
|
);
|
|
@@ -164,10 +224,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
164
224
|
next_after: '3'
|
|
165
225
|
});
|
|
166
226
|
|
|
167
|
-
const batch3 = await test_utils.
|
|
227
|
+
const batch3 = await test_utils.getBatchArray(
|
|
168
228
|
bucketStorage.getBucketDataBatch(
|
|
169
|
-
checkpoint,
|
|
170
|
-
[bucketRequest(
|
|
229
|
+
test_utils.testCheckpoint(checkpoint),
|
|
230
|
+
[bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
|
|
171
231
|
options
|
|
172
232
|
)
|
|
173
233
|
);
|
|
@@ -189,7 +249,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
189
249
|
await Promise.all(
|
|
190
250
|
(
|
|
191
251
|
await mongoFactory.db.db
|
|
192
|
-
.listCollections(
|
|
252
|
+
.listCollections(
|
|
253
|
+
{ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) },
|
|
254
|
+
{ nameOnly: true }
|
|
255
|
+
)
|
|
193
256
|
.toArray()
|
|
194
257
|
).map((collection: { name: string }) =>
|
|
195
258
|
mongoFactory.db.db
|
|
@@ -388,8 +451,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
388
451
|
}
|
|
389
452
|
);
|
|
390
453
|
|
|
391
|
-
test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions
|
|
392
|
-
// Tests the behavior of resolveTables when bucket data sources and parameter index creators are added
|
|
454
|
+
test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions', async () => {
|
|
455
|
+
// Tests the behavior of resolveTables when bucket data sources and parameter index creators are added.
|
|
393
456
|
// These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
|
|
394
457
|
// This just tests the specific resolveTables behavior.
|
|
395
458
|
|
|
@@ -434,20 +497,19 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
434
497
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
|
|
435
498
|
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
436
499
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
437
|
-
const fullRules =
|
|
438
|
-
const dataOnlyRules =
|
|
439
|
-
const parameterOnlyRules =
|
|
440
|
-
const eventOnlyRules =
|
|
500
|
+
const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
|
|
501
|
+
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
|
|
502
|
+
const parameterOnlyRules = parsedSyncConfigSetFor(parameterOnlyRulesYaml, storageVersion);
|
|
503
|
+
const eventOnlyRules = parsedSyncConfigSetFor(eventOnlyRulesYaml, storageVersion);
|
|
441
504
|
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
442
505
|
const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
|
|
443
506
|
const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
|
|
444
|
-
const removedDataTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
|
|
445
507
|
|
|
446
508
|
const dataOnly = await writer.resolveTables({
|
|
447
509
|
connection_id: 1,
|
|
448
510
|
source,
|
|
449
511
|
idGenerator: () => dataOnlyTableId,
|
|
450
|
-
|
|
512
|
+
parsedSyncConfig: dataOnlyRules
|
|
451
513
|
});
|
|
452
514
|
expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
453
515
|
expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
@@ -458,7 +520,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
458
520
|
connection_id: 1,
|
|
459
521
|
source,
|
|
460
522
|
idGenerator: () => addedParameterTableId,
|
|
461
|
-
|
|
523
|
+
parsedSyncConfig: fullRules
|
|
462
524
|
});
|
|
463
525
|
// Adding a definition always creates a new SourceTable
|
|
464
526
|
expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
|
|
@@ -472,29 +534,29 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
472
534
|
idGenerator: () => {
|
|
473
535
|
throw new Error('data-only resolve should reuse existing v3 source table');
|
|
474
536
|
},
|
|
475
|
-
|
|
537
|
+
parsedSyncConfig: dataOnlyRules
|
|
476
538
|
});
|
|
477
539
|
expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
478
|
-
//
|
|
479
|
-
expect(removedParameter.dropTables.map((table) => table.id)).toEqual([
|
|
540
|
+
// Same-identity tables that no longer cover desired memberships are left in place.
|
|
541
|
+
expect(removedParameter.dropTables.map((table) => table.id)).toEqual([]);
|
|
480
542
|
expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
|
|
481
543
|
expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
|
|
482
|
-
await writer.drop(removedParameter.dropTables);
|
|
483
544
|
|
|
484
545
|
const removedData = await writer.resolveTables({
|
|
485
546
|
connection_id: 1,
|
|
486
547
|
source,
|
|
487
|
-
idGenerator: () =>
|
|
488
|
-
|
|
548
|
+
idGenerator: () => {
|
|
549
|
+
throw new Error('parameter-only resolve should reuse existing v3 source table');
|
|
550
|
+
},
|
|
551
|
+
parsedSyncConfig: parameterOnlyRules
|
|
489
552
|
});
|
|
490
553
|
|
|
491
|
-
// This goes from dataOnlyRules -> parameterOnlyRules, which
|
|
492
|
-
//
|
|
493
|
-
expect(removedData.tables.map((table) => table.id)).toEqual([
|
|
494
|
-
expect(removedData.dropTables.map((table) => table.id)).toEqual([
|
|
554
|
+
// This goes from dataOnlyRules -> parameterOnlyRules, which removes one definition but
|
|
555
|
+
// reuses the existing parameter-only source table.
|
|
556
|
+
expect(removedData.tables.map((table) => table.id)).toEqual([addedParameterTableId]);
|
|
557
|
+
expect(removedData.dropTables.map((table) => table.id)).toEqual([]);
|
|
495
558
|
expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
|
|
496
559
|
expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
|
|
497
|
-
await writer.drop(removedData.dropTables);
|
|
498
560
|
|
|
499
561
|
const eventOnly = await writer.resolveTables({
|
|
500
562
|
connection_id: 1,
|
|
@@ -502,11 +564,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
502
564
|
idGenerator: () => {
|
|
503
565
|
throw new Error('resolve should reuse existing v3 source table');
|
|
504
566
|
},
|
|
505
|
-
|
|
567
|
+
parsedSyncConfig: eventOnlyRules
|
|
506
568
|
});
|
|
507
569
|
|
|
508
570
|
// Event-only table can re-use any existing table.
|
|
509
|
-
expect(eventOnly.tables
|
|
571
|
+
expect(eventOnly.tables).toHaveLength(1);
|
|
572
|
+
expect([dataOnlyTableId.toString(), addedParameterTableId.toString()]).toContain(eventOnly.tables[0].id.toString());
|
|
510
573
|
expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
511
574
|
expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
|
|
512
575
|
expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
|
|
@@ -515,6 +578,76 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
515
578
|
expect(eventOnly.tables[0].syncEvent).toBe(true);
|
|
516
579
|
});
|
|
517
580
|
|
|
581
|
+
test.runIf(storageVersion >= 3)(
|
|
582
|
+
'resolveTables designates a single event carrier across split source tables',
|
|
583
|
+
async () => {
|
|
584
|
+
// When memberships are split over multiple SourceTables for the same ref, a row change
|
|
585
|
+
// is saved once per table. Only one table may fire events, otherwise the same event
|
|
586
|
+
// would fire once per table for every row change.
|
|
587
|
+
const dataOnlyEventYaml = `
|
|
588
|
+
bucket_definitions:
|
|
589
|
+
by_owner:
|
|
590
|
+
parameters:
|
|
591
|
+
- SELECT token_parameters.owner_id as owner_id
|
|
592
|
+
data:
|
|
593
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
594
|
+
|
|
595
|
+
event_definitions:
|
|
596
|
+
write_checkpoints:
|
|
597
|
+
payloads:
|
|
598
|
+
- SELECT id, owner_id FROM memberships
|
|
599
|
+
`;
|
|
600
|
+
const fullEventYaml = `
|
|
601
|
+
bucket_definitions:
|
|
602
|
+
by_owner:
|
|
603
|
+
parameters:
|
|
604
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
605
|
+
data:
|
|
606
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
607
|
+
|
|
608
|
+
event_definitions:
|
|
609
|
+
write_checkpoints:
|
|
610
|
+
payloads:
|
|
611
|
+
- SELECT id, owner_id FROM memberships
|
|
612
|
+
`;
|
|
613
|
+
|
|
614
|
+
await using factory = await storageConfig.factory();
|
|
615
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
|
|
616
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
617
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
618
|
+
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
|
|
619
|
+
const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
|
|
620
|
+
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
621
|
+
|
|
622
|
+
const initial = await writer.resolveTables({
|
|
623
|
+
connection_id: 1,
|
|
624
|
+
source,
|
|
625
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
|
|
626
|
+
parsedSyncConfig: dataOnlyRules
|
|
627
|
+
});
|
|
628
|
+
expect(initial.tables).toHaveLength(1);
|
|
629
|
+
expect(initial.tables[0].syncEvent).toBe(true);
|
|
630
|
+
|
|
631
|
+
// Adding the table-based parameter lookup creates a second SourceTable for the same ref.
|
|
632
|
+
const split = await writer.resolveTables({
|
|
633
|
+
connection_id: 1,
|
|
634
|
+
source,
|
|
635
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
|
|
636
|
+
parsedSyncConfig: fullRules
|
|
637
|
+
});
|
|
638
|
+
expect(split.tables).toHaveLength(2);
|
|
639
|
+
// Both tables match the event by ref, but only one may carry it.
|
|
640
|
+
const carriers = split.tables.filter((table) => table.syncEvent);
|
|
641
|
+
expect(carriers).toHaveLength(1);
|
|
642
|
+
|
|
643
|
+
// getSourceTableStatus preserves the carrier designation rather than recomputing it
|
|
644
|
+
// from the ref, so refreshing a non-carrier table does not make it fire events.
|
|
645
|
+
const nonCarrier = split.tables.find((table) => !table.syncEvent)!;
|
|
646
|
+
const refreshed = await writer.getSourceTableStatus(nonCarrier);
|
|
647
|
+
expect(refreshed!.syncEvent).toBe(false);
|
|
648
|
+
}
|
|
649
|
+
);
|
|
650
|
+
|
|
518
651
|
test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
|
|
519
652
|
await using factory = await storageConfig.factory();
|
|
520
653
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -531,7 +664,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
531
664
|
)
|
|
532
665
|
);
|
|
533
666
|
const bucketStorage = factory.getInstance(syncRules);
|
|
534
|
-
const
|
|
667
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
668
|
+
const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
|
|
535
669
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
536
670
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
537
671
|
|
|
@@ -552,7 +686,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
552
686
|
const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
|
|
553
687
|
const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
|
|
554
688
|
const buckets = await querier.queryDynamicBucketDescriptions({
|
|
555
|
-
async getParameterSets(lookups) {
|
|
689
|
+
async getParameterSets(lookups: ScopedParameterLookup[]) {
|
|
556
690
|
expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
|
|
557
691
|
expect(lookups[0].indexId).toEqual('1');
|
|
558
692
|
|
|
@@ -561,33 +695,635 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
561
695
|
return parameter_sets;
|
|
562
696
|
}
|
|
563
697
|
});
|
|
564
|
-
expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(
|
|
698
|
+
expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]);
|
|
565
699
|
|
|
566
700
|
const mongoFactory = factory as MongoBucketStorage;
|
|
567
701
|
const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
|
|
568
|
-
const currentDataCollections = await db.
|
|
702
|
+
const currentDataCollections = await db.listSourceRecordCollections(syncRules.replicationStreamId);
|
|
569
703
|
const currentData = await currentDataCollections[0]?.findOne({});
|
|
570
704
|
const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
|
|
571
705
|
expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
|
|
572
706
|
|
|
573
707
|
const bucketCollections = await mongoFactory.db.db
|
|
574
|
-
.listCollections({ name: new RegExp(`^bucket_data_${syncRules.
|
|
708
|
+
.listCollections({ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
|
|
575
709
|
.toArray();
|
|
576
710
|
expect(
|
|
577
|
-
bucketCollections.some(
|
|
711
|
+
bucketCollections.some(
|
|
712
|
+
(collection) => collection.name === `bucket_data_${syncRules.replicationStreamId}_${firstBucket?.def}`
|
|
713
|
+
)
|
|
578
714
|
).toBe(true);
|
|
579
715
|
|
|
580
|
-
const syncRule = (await mongoFactory.db.sync_rules.findOne({
|
|
716
|
+
const syncRule = (await mongoFactory.db.sync_rules.findOne({
|
|
717
|
+
_id: syncRules.replicationStreamId
|
|
718
|
+
})) as ReplicationStreamDocumentV3;
|
|
581
719
|
const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
|
|
582
720
|
const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
|
|
583
721
|
expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
|
|
584
722
|
|
|
585
723
|
const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
|
|
586
724
|
expect(parameterIndexId).toBeDefined();
|
|
587
|
-
const parameterEntry = await db.
|
|
725
|
+
const parameterEntry = await db.parameterIndex(syncRules.replicationStreamId, parameterIndexId!).findOne({});
|
|
588
726
|
expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
|
|
589
727
|
});
|
|
590
728
|
|
|
729
|
+
test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => {
|
|
730
|
+
await using factory = await storageConfig.factory();
|
|
731
|
+
|
|
732
|
+
const first = await factory.updateSyncRules(
|
|
733
|
+
updateSyncRulesFromYaml(
|
|
734
|
+
`
|
|
735
|
+
config:
|
|
736
|
+
edition: 2
|
|
737
|
+
|
|
738
|
+
streams:
|
|
739
|
+
by_owner:
|
|
740
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
741
|
+
`,
|
|
742
|
+
{ storageVersion }
|
|
743
|
+
)
|
|
744
|
+
);
|
|
745
|
+
const second = await factory.updateSyncRules(
|
|
746
|
+
updateSyncRulesFromYaml(
|
|
747
|
+
`
|
|
748
|
+
config:
|
|
749
|
+
edition: 2
|
|
750
|
+
|
|
751
|
+
streams:
|
|
752
|
+
by_project:
|
|
753
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
754
|
+
`,
|
|
755
|
+
{ storageVersion }
|
|
756
|
+
)
|
|
757
|
+
);
|
|
758
|
+
|
|
759
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
760
|
+
expect(
|
|
761
|
+
(await factory.getStoppedReplicationStreams()).find(
|
|
762
|
+
(stream) => stream.replicationStreamId == first.replicationStreamId
|
|
763
|
+
)?.state
|
|
764
|
+
).toBe(storage.SyncRuleState.STOP);
|
|
765
|
+
|
|
766
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
767
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
768
|
+
expect(replicatingStreams[0].replicationStreamId).toEqual(second.replicationStreamId);
|
|
769
|
+
|
|
770
|
+
const configs = second.syncConfigContent;
|
|
771
|
+
expect(configs).toHaveLength(1);
|
|
772
|
+
const statuses = await Promise.all(configs.map((config) => config.getSyncConfigStatus()));
|
|
773
|
+
expect(statuses.map((status) => status?.state)).toEqual([storage.SyncRuleState.PROCESSING]);
|
|
774
|
+
expect(statuses.map((status) => status?.id)).toEqual(configs.map((config) => config.syncConfigId));
|
|
775
|
+
const parsed = replicatingStreams[0].parsed(test_utils.PARSE_OPTIONS);
|
|
776
|
+
expect(parsed.syncConfigs).toHaveLength(1);
|
|
777
|
+
expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
|
|
778
|
+
|
|
779
|
+
const bucketStorage = factory.getInstance(replicatingStreams[0]);
|
|
780
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
781
|
+
const resolved = await writer.resolveTables({
|
|
782
|
+
connection_id: 1,
|
|
783
|
+
source: sourceDescriptor('todos', { objectId: 'todos-relation' }),
|
|
784
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834b')
|
|
785
|
+
});
|
|
786
|
+
expect(resolved.tables).toHaveLength(1);
|
|
787
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
|
|
788
|
+
});
|
|
789
|
+
|
|
790
|
+
test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => {
|
|
791
|
+
await using factory = await storageConfig.factory();
|
|
792
|
+
|
|
793
|
+
const first = await factory.updateSyncRules(
|
|
794
|
+
updateSyncRulesFromYaml(
|
|
795
|
+
`
|
|
796
|
+
config:
|
|
797
|
+
edition: 3
|
|
798
|
+
|
|
799
|
+
streams:
|
|
800
|
+
by_owner:
|
|
801
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
802
|
+
`,
|
|
803
|
+
{ storageVersion }
|
|
804
|
+
)
|
|
805
|
+
);
|
|
806
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
807
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
808
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
809
|
+
await firstWriter.commit('1/1');
|
|
810
|
+
|
|
811
|
+
const second = await factory.updateSyncRules(
|
|
812
|
+
updateSyncRulesFromYaml(
|
|
813
|
+
`
|
|
814
|
+
config:
|
|
815
|
+
edition: 3
|
|
816
|
+
|
|
817
|
+
streams:
|
|
818
|
+
by_project:
|
|
819
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
820
|
+
`,
|
|
821
|
+
{ storageVersion }
|
|
822
|
+
)
|
|
823
|
+
);
|
|
824
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
825
|
+
|
|
826
|
+
const bucketStorage = factory.getInstance(second);
|
|
827
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
828
|
+
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
829
|
+
|
|
830
|
+
const resolved = await writer.resolveTables({
|
|
831
|
+
connection_id: 1,
|
|
832
|
+
source,
|
|
833
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834c')
|
|
834
|
+
});
|
|
835
|
+
expect(resolved.tables).toHaveLength(1);
|
|
836
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(2);
|
|
837
|
+
|
|
838
|
+
await writer.markAllSnapshotDone('2/1');
|
|
839
|
+
await writer.commit('2/1');
|
|
840
|
+
|
|
841
|
+
const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
842
|
+
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
843
|
+
const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
|
|
844
|
+
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
test.runIf(storageVersion >= 3)(
|
|
848
|
+
'editing a stream query on the same table does not drop the active source table',
|
|
849
|
+
async () => {
|
|
850
|
+
await using factory = await storageConfig.factory();
|
|
851
|
+
|
|
852
|
+
const first = await factory.updateSyncRules(
|
|
853
|
+
updateSyncRulesFromYaml(
|
|
854
|
+
`
|
|
855
|
+
config:
|
|
856
|
+
edition: 3
|
|
857
|
+
|
|
858
|
+
streams:
|
|
859
|
+
todos:
|
|
860
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
861
|
+
`,
|
|
862
|
+
{ storageVersion }
|
|
863
|
+
)
|
|
864
|
+
);
|
|
865
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
866
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
867
|
+
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
868
|
+
const firstResolved = await firstWriter.resolveTables({
|
|
869
|
+
connection_id: 1,
|
|
870
|
+
source,
|
|
871
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
|
|
872
|
+
});
|
|
873
|
+
expect(firstResolved.dropTables).toHaveLength(0);
|
|
874
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
875
|
+
await firstWriter.commit('1/1');
|
|
876
|
+
|
|
877
|
+
const second = await factory.updateSyncRules(
|
|
878
|
+
updateSyncRulesFromYaml(
|
|
879
|
+
`
|
|
880
|
+
config:
|
|
881
|
+
edition: 3
|
|
882
|
+
|
|
883
|
+
streams:
|
|
884
|
+
todos:
|
|
885
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
886
|
+
`,
|
|
887
|
+
{ storageVersion }
|
|
888
|
+
)
|
|
889
|
+
);
|
|
890
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
891
|
+
|
|
892
|
+
const deploying = await factory.getDeployingSyncConfig();
|
|
893
|
+
expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
|
|
894
|
+
const bucketStorage = deploying!.storage;
|
|
895
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
896
|
+
const resolved = await writer.resolveTables({
|
|
897
|
+
connection_id: 1,
|
|
898
|
+
source,
|
|
899
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834e')
|
|
900
|
+
});
|
|
901
|
+
|
|
902
|
+
expect(resolved.dropTables).toHaveLength(0);
|
|
903
|
+
expect(resolved.tables.map((table) => table.id.toString())).toEqual(['6544e3899293153fa7b3834e']);
|
|
904
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
|
|
905
|
+
|
|
906
|
+
const activeStatus = await firstWriter.getSourceTableStatus(firstResolved.tables[0]);
|
|
907
|
+
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
908
|
+
}
|
|
909
|
+
);
|
|
910
|
+
|
|
911
|
+
test.runIf(storageVersion >= 3)('reserves historical mapping ids without reusing stopped configs', async () => {
|
|
912
|
+
await using factory = await storageConfig.factory();
|
|
913
|
+
|
|
914
|
+
const ownerRules = `
|
|
915
|
+
config:
|
|
916
|
+
edition: 3
|
|
917
|
+
|
|
918
|
+
streams:
|
|
919
|
+
by_owner:
|
|
920
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
921
|
+
`;
|
|
922
|
+
const projectRules = `
|
|
923
|
+
config:
|
|
924
|
+
edition: 3
|
|
925
|
+
|
|
926
|
+
streams:
|
|
927
|
+
by_project:
|
|
928
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
929
|
+
`;
|
|
930
|
+
const statusRules = `
|
|
931
|
+
config:
|
|
932
|
+
edition: 3
|
|
933
|
+
|
|
934
|
+
streams:
|
|
935
|
+
by_status:
|
|
936
|
+
query: SELECT * FROM todos WHERE status = subscription.parameter('status')
|
|
937
|
+
`;
|
|
938
|
+
|
|
939
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
|
|
940
|
+
const firstStorage = factory.getInstance(first);
|
|
941
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
942
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
943
|
+
await firstWriter.commit('1/1');
|
|
944
|
+
|
|
945
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(projectRules, { storageVersion }));
|
|
946
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
947
|
+
|
|
948
|
+
const third = await factory.updateSyncRules(updateSyncRulesFromYaml(statusRules, { storageVersion }));
|
|
949
|
+
expect(third.replicationStreamId).toBe(first.replicationStreamId);
|
|
950
|
+
|
|
951
|
+
const configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
952
|
+
const statuses = await Promise.all(
|
|
953
|
+
configs.map(async (config) => [config, await config.getSyncConfigStatus()] as const)
|
|
954
|
+
);
|
|
955
|
+
// ownerRules
|
|
956
|
+
const activeConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.ACTIVE)?.[0];
|
|
957
|
+
// projectRules
|
|
958
|
+
const stoppedConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.STOP)?.[0];
|
|
959
|
+
// statusRules
|
|
960
|
+
const processingConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.PROCESSING)?.[0];
|
|
961
|
+
expect(stoppedConfig).toBeDefined();
|
|
962
|
+
expect(activeConfig).toBeDefined();
|
|
963
|
+
expect(processingConfig).toBeDefined();
|
|
964
|
+
|
|
965
|
+
const activePrefix = bucketRequest(activeConfig!, 'by_owner|0["owner"]').bucket;
|
|
966
|
+
const stoppedPrefix = bucketRequest(stoppedConfig!, 'by_project|0["project"]').bucket;
|
|
967
|
+
const processingPrefix = bucketRequest(processingConfig!, 'by_status|0["status"]').bucket;
|
|
968
|
+
|
|
969
|
+
expect(processingPrefix).not.toBe(stoppedPrefix);
|
|
970
|
+
expect(processingPrefix).not.toBe(activePrefix);
|
|
971
|
+
expect(bucketDefinitionId(processingPrefix)).toBeGreaterThan(
|
|
972
|
+
Math.max(bucketDefinitionId(stoppedPrefix), bucketDefinitionId(activePrefix))
|
|
973
|
+
);
|
|
974
|
+
});
|
|
975
|
+
|
|
976
|
+
test.runIf(storageVersion >= 3)('table snapshot status only affects sync configs using that table', async () => {
|
|
977
|
+
await using factory = await storageConfig.factory();
|
|
978
|
+
|
|
979
|
+
const first = await factory.updateSyncRules(
|
|
980
|
+
updateSyncRulesFromYaml(
|
|
981
|
+
`
|
|
982
|
+
config:
|
|
983
|
+
edition: 3
|
|
984
|
+
|
|
985
|
+
streams:
|
|
986
|
+
by_owner:
|
|
987
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
988
|
+
`,
|
|
989
|
+
{ storageVersion }
|
|
990
|
+
)
|
|
991
|
+
);
|
|
992
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
993
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
994
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
995
|
+
await firstWriter.commit('1/1');
|
|
996
|
+
|
|
997
|
+
const second = await factory.updateSyncRules(
|
|
998
|
+
updateSyncRulesFromYaml(
|
|
999
|
+
`
|
|
1000
|
+
config:
|
|
1001
|
+
edition: 3
|
|
1002
|
+
|
|
1003
|
+
streams:
|
|
1004
|
+
by_owner:
|
|
1005
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1006
|
+
by_project:
|
|
1007
|
+
query: SELECT * FROM projects WHERE id = subscription.parameter('project_id')
|
|
1008
|
+
`,
|
|
1009
|
+
{ storageVersion }
|
|
1010
|
+
)
|
|
1011
|
+
);
|
|
1012
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1013
|
+
|
|
1014
|
+
const bucketStorage = factory.getInstance(second);
|
|
1015
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1016
|
+
const resolved = await writer.resolveTables({
|
|
1017
|
+
connection_id: 1,
|
|
1018
|
+
source: sourceDescriptor('projects', { objectId: 'projects-relation' }),
|
|
1019
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
|
|
1020
|
+
});
|
|
1021
|
+
expect(resolved.tables).toHaveLength(1);
|
|
1022
|
+
|
|
1023
|
+
await writer.markAllSnapshotDone('2/1');
|
|
1024
|
+
await writer.markTableSnapshotRequired(resolved.tables[0]);
|
|
1025
|
+
|
|
1026
|
+
const configsAfterRequired = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1027
|
+
const statusesAfterRequired = await Promise.all(configsAfterRequired.map((config) => config.getSyncConfigStatus()));
|
|
1028
|
+
const ownerConfigAfterRequired = statusesAfterRequired.find(
|
|
1029
|
+
(status) => status?.state == storage.SyncRuleState.ACTIVE
|
|
1030
|
+
);
|
|
1031
|
+
const projectConfigAfterRequired = statusesAfterRequired.find(
|
|
1032
|
+
(status) => status?.state == storage.SyncRuleState.PROCESSING
|
|
1033
|
+
);
|
|
1034
|
+
expect(ownerConfigAfterRequired?.snapshot_done).toBe(true);
|
|
1035
|
+
expect(projectConfigAfterRequired?.snapshot_done).toBe(false);
|
|
1036
|
+
|
|
1037
|
+
await writer.markTableSnapshotDone(resolved.tables, '3/1');
|
|
1038
|
+
|
|
1039
|
+
const configsAfterDone = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1040
|
+
const statusesAfterDone = await Promise.all(configsAfterDone.map((config) => config.getSyncConfigStatus()));
|
|
1041
|
+
const ownerConfigAfterDone = statusesAfterDone.find((status) => status?.id == ownerConfigAfterRequired!.id);
|
|
1042
|
+
const projectConfigAfterDone = statusesAfterDone.find((status) => status?.id == projectConfigAfterRequired!.id);
|
|
1043
|
+
expect(ownerConfigAfterDone?.snapshot_done).toBe(true);
|
|
1044
|
+
expect(projectConfigAfterDone?.snapshot_done).toBe(false);
|
|
1045
|
+
});
|
|
1046
|
+
|
|
1047
|
+
test.runIf(storageVersion >= 3)(
|
|
1048
|
+
'keeps compatible active and deploying sync configs in one replication stream',
|
|
1049
|
+
async () => {
|
|
1050
|
+
await using factory = await storageConfig.factory();
|
|
1051
|
+
|
|
1052
|
+
const first = await factory.updateSyncRules(
|
|
1053
|
+
updateSyncRulesFromYaml(
|
|
1054
|
+
`
|
|
1055
|
+
config:
|
|
1056
|
+
edition: 3
|
|
1057
|
+
|
|
1058
|
+
streams:
|
|
1059
|
+
by_owner:
|
|
1060
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1061
|
+
`,
|
|
1062
|
+
{ storageVersion }
|
|
1063
|
+
)
|
|
1064
|
+
);
|
|
1065
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1066
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1067
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1068
|
+
await firstWriter.commit('1/1');
|
|
1069
|
+
const initialActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1070
|
+
|
|
1071
|
+
let configs = first.syncConfigContent;
|
|
1072
|
+
const firstConfigId = configs[0].syncConfigId;
|
|
1073
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
|
|
1074
|
+
|
|
1075
|
+
const second = await factory.updateSyncRules(
|
|
1076
|
+
updateSyncRulesFromYaml(
|
|
1077
|
+
`
|
|
1078
|
+
config:
|
|
1079
|
+
edition: 3
|
|
1080
|
+
|
|
1081
|
+
streams:
|
|
1082
|
+
by_project:
|
|
1083
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1084
|
+
`,
|
|
1085
|
+
{ storageVersion }
|
|
1086
|
+
)
|
|
1087
|
+
);
|
|
1088
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1089
|
+
|
|
1090
|
+
configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1091
|
+
const deploying = await factory.getDeployingSyncConfig();
|
|
1092
|
+
expect(configs).toHaveLength(2);
|
|
1093
|
+
expect(deploying).not.toBeNull();
|
|
1094
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
|
|
1095
|
+
|
|
1096
|
+
const stream = (await factory.getActiveSyncConfig())?.replicationStream;
|
|
1097
|
+
expect(stream?.state).toBe(storage.SyncRuleState.ACTIVE);
|
|
1098
|
+
expect(
|
|
1099
|
+
(await Promise.all(configs.map((config) => config.getSyncConfigStatus()))).map((status) => status?.state).sort()
|
|
1100
|
+
).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING]);
|
|
1101
|
+
|
|
1102
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1103
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1104
|
+
for (const config of configs) {
|
|
1105
|
+
expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
|
|
1109
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1110
|
+
await secondWriter.markAllSnapshotDone('2/1');
|
|
1111
|
+
await secondWriter.commit('2/1');
|
|
1112
|
+
|
|
1113
|
+
const updatedStream = (await factory.getActiveSyncConfig())?.replicationStream;
|
|
1114
|
+
expect(updatedStream?.state).toBe(storage.SyncRuleState.ACTIVE);
|
|
1115
|
+
const updatedConfigs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1116
|
+
expect(
|
|
1117
|
+
(await Promise.all(updatedConfigs.map((config) => config.getSyncConfigStatus())))
|
|
1118
|
+
.map((status) => status?.state)
|
|
1119
|
+
.sort()
|
|
1120
|
+
).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.STOP]);
|
|
1121
|
+
expect(await factory.getDeployingSyncConfig()).toBeNull();
|
|
1122
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).not.toBe(firstConfigId);
|
|
1123
|
+
const updatedActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1124
|
+
expect(updatedActiveStorage.replicationStream.replicationJobId).not.toBe(
|
|
1125
|
+
initialActiveStorage.replicationStream.replicationJobId
|
|
1126
|
+
);
|
|
1127
|
+
expect(updatedActiveStorage.replicationStream.replicationJobId).toContain(
|
|
1128
|
+
(await factory.getActiveSyncConfig())!.content.syncConfigId
|
|
1129
|
+
);
|
|
1130
|
+
}
|
|
1131
|
+
);
|
|
1132
|
+
|
|
1133
|
+
test.runIf(storageVersion >= 3)(
|
|
1134
|
+
'appended compatible config adopts the stream checkpoint instead of regressing to 0',
|
|
1135
|
+
async () => {
|
|
1136
|
+
await using factory = await storageConfig.factory();
|
|
1137
|
+
|
|
1138
|
+
// First config replicates some data, advancing the stream-level op head and its checkpoint.
|
|
1139
|
+
const first = await factory.updateSyncRules(
|
|
1140
|
+
updateSyncRulesFromYaml(
|
|
1141
|
+
`
|
|
1142
|
+
config:
|
|
1143
|
+
edition: 3
|
|
1144
|
+
|
|
1145
|
+
streams:
|
|
1146
|
+
by_owner:
|
|
1147
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1148
|
+
`,
|
|
1149
|
+
{ storageVersion }
|
|
1150
|
+
)
|
|
1151
|
+
);
|
|
1152
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1153
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1154
|
+
const sourceTable = await test_utils.resolveTestTable(
|
|
1155
|
+
firstWriter,
|
|
1156
|
+
'todos',
|
|
1157
|
+
['id'],
|
|
1158
|
+
INITIALIZED_MONGO_STORAGE_FACTORY
|
|
1159
|
+
);
|
|
1160
|
+
await firstWriter.save({
|
|
1161
|
+
sourceTable,
|
|
1162
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1163
|
+
after: {
|
|
1164
|
+
id: 'todo-1',
|
|
1165
|
+
owner_id: 'user-1',
|
|
1166
|
+
project_id: 'project-1'
|
|
1167
|
+
},
|
|
1168
|
+
afterReplicaId: test_utils.rid('todo-1')
|
|
1169
|
+
});
|
|
1170
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1171
|
+
await firstWriter.commit('1/1');
|
|
1172
|
+
|
|
1173
|
+
const firstCheckpoint = (await firstStorage.getCheckpoint()).checkpoint;
|
|
1174
|
+
expect(firstCheckpoint).toBeGreaterThan(0n);
|
|
1175
|
+
|
|
1176
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1177
|
+
const streamDocBefore = (await mongoFactory.db.sync_rules.findOne({
|
|
1178
|
+
_id: first.replicationStreamId
|
|
1179
|
+
})) as ReplicationStreamDocumentV3;
|
|
1180
|
+
// The stream-level head was advanced durably to (at least) the first config's checkpoint.
|
|
1181
|
+
expect(streamDocBefore.last_persisted_op).not.toBeNull();
|
|
1182
|
+
expect(BigInt(streamDocBefore.last_persisted_op!)).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1183
|
+
|
|
1184
|
+
// Append a compatible second config that replicates nothing new.
|
|
1185
|
+
const second = await factory.updateSyncRules(
|
|
1186
|
+
updateSyncRulesFromYaml(
|
|
1187
|
+
`
|
|
1188
|
+
config:
|
|
1189
|
+
edition: 3
|
|
1190
|
+
|
|
1191
|
+
streams:
|
|
1192
|
+
by_project:
|
|
1193
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1194
|
+
`,
|
|
1195
|
+
{ storageVersion }
|
|
1196
|
+
)
|
|
1197
|
+
);
|
|
1198
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1199
|
+
|
|
1200
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1201
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1202
|
+
const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
|
|
1203
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1204
|
+
// No new data replicated - just complete the snapshot and commit.
|
|
1205
|
+
await secondWriter.markAllSnapshotDone('2/1');
|
|
1206
|
+
await secondWriter.commit('2/1');
|
|
1207
|
+
|
|
1208
|
+
// The appended config's checkpoint must adopt the stream head, not regress to 0.
|
|
1209
|
+
const streamDocAfter = (await mongoFactory.db.sync_rules.findOne({
|
|
1210
|
+
_id: first.replicationStreamId
|
|
1211
|
+
})) as ReplicationStreamDocumentV3;
|
|
1212
|
+
const head = BigInt(streamDocAfter.last_persisted_op!);
|
|
1213
|
+
for (const config of streamDocAfter.sync_configs) {
|
|
1214
|
+
expect(config.last_checkpoint).not.toBeNull();
|
|
1215
|
+
expect(BigInt(config.last_checkpoint!)).toEqual(head);
|
|
1216
|
+
}
|
|
1217
|
+
// All configs share the same checkpoint, equal to the stream head.
|
|
1218
|
+
expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1219
|
+
|
|
1220
|
+
// After activation, the active config's checkpoint does not regress.
|
|
1221
|
+
const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1222
|
+
const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
|
|
1223
|
+
expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1224
|
+
}
|
|
1225
|
+
);
|
|
1226
|
+
|
|
1227
|
+
test.runIf(storageVersion >= 3)('creates a new replication stream when compatibility options differ', async () => {
|
|
1228
|
+
await using factory = await storageConfig.factory();
|
|
1229
|
+
|
|
1230
|
+
const first = await factory.updateSyncRules(
|
|
1231
|
+
updateSyncRulesFromYaml(
|
|
1232
|
+
`
|
|
1233
|
+
config:
|
|
1234
|
+
edition: 3
|
|
1235
|
+
|
|
1236
|
+
streams:
|
|
1237
|
+
by_owner:
|
|
1238
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1239
|
+
`,
|
|
1240
|
+
{ storageVersion }
|
|
1241
|
+
)
|
|
1242
|
+
);
|
|
1243
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1244
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1245
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1246
|
+
await firstWriter.commit('1/1');
|
|
1247
|
+
|
|
1248
|
+
// Same streams, but a different compatibility edition - must not append to the active stream.
|
|
1249
|
+
const second = await factory.updateSyncRules(
|
|
1250
|
+
updateSyncRulesFromYaml(
|
|
1251
|
+
`
|
|
1252
|
+
config:
|
|
1253
|
+
edition: 2
|
|
1254
|
+
|
|
1255
|
+
streams:
|
|
1256
|
+
by_owner:
|
|
1257
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1258
|
+
`,
|
|
1259
|
+
{ storageVersion }
|
|
1260
|
+
)
|
|
1261
|
+
);
|
|
1262
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1263
|
+
|
|
1264
|
+
// The first stream stays active until the replacement has replicated.
|
|
1265
|
+
const active = await factory.getActiveSyncConfig();
|
|
1266
|
+
expect(active?.replicationStream.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1267
|
+
expect((await factory.getDeployingSyncConfig())?.replicationStream.replicationStreamId).toEqual(
|
|
1268
|
+
second.replicationStreamId
|
|
1269
|
+
);
|
|
1270
|
+
});
|
|
1271
|
+
|
|
1272
|
+
test.runIf(storageVersion >= 3)(
|
|
1273
|
+
'creates a new replication stream for legacy sync rules without a serialized plan',
|
|
1274
|
+
async () => {
|
|
1275
|
+
await using factory = await storageConfig.factory();
|
|
1276
|
+
|
|
1277
|
+
const first = await factory.updateSyncRules(
|
|
1278
|
+
updateSyncRulesFromYaml(
|
|
1279
|
+
`
|
|
1280
|
+
config:
|
|
1281
|
+
edition: 3
|
|
1282
|
+
|
|
1283
|
+
streams:
|
|
1284
|
+
by_owner:
|
|
1285
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1286
|
+
`,
|
|
1287
|
+
{ storageVersion }
|
|
1288
|
+
)
|
|
1289
|
+
);
|
|
1290
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1291
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1292
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1293
|
+
await firstWriter.commit('1/1');
|
|
1294
|
+
|
|
1295
|
+
// Legacy sync rules have no serialized plan - must not append to the active stream.
|
|
1296
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1297
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1298
|
+
}
|
|
1299
|
+
);
|
|
1300
|
+
|
|
1301
|
+
test.runIf(storageVersion >= 3)('does not append to an active legacy sync config', async () => {
|
|
1302
|
+
await using factory = await storageConfig.factory();
|
|
1303
|
+
|
|
1304
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1305
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1306
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1307
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1308
|
+
await firstWriter.commit('1/1');
|
|
1309
|
+
|
|
1310
|
+
// The active config has no serialized plan, so a new streams config must not be appended to it.
|
|
1311
|
+
const second = await factory.updateSyncRules(
|
|
1312
|
+
updateSyncRulesFromYaml(
|
|
1313
|
+
`
|
|
1314
|
+
config:
|
|
1315
|
+
edition: 3
|
|
1316
|
+
|
|
1317
|
+
streams:
|
|
1318
|
+
by_owner:
|
|
1319
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1320
|
+
`,
|
|
1321
|
+
{ storageVersion }
|
|
1322
|
+
)
|
|
1323
|
+
);
|
|
1324
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1325
|
+
});
|
|
1326
|
+
|
|
591
1327
|
test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
|
|
592
1328
|
await using factory = await storageConfig.factory();
|
|
593
1329
|
|
|
@@ -600,7 +1336,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
600
1336
|
).resolves.toBeDefined();
|
|
601
1337
|
|
|
602
1338
|
const mongoFactory = factory as MongoBucketStorage;
|
|
603
|
-
expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.
|
|
1339
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.replicationStreamId }))?.state).toBe(
|
|
604
1340
|
storage.SyncRuleState.STOP
|
|
605
1341
|
);
|
|
606
1342
|
});
|
|
@@ -612,10 +1348,48 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
612
1348
|
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
|
|
613
1349
|
);
|
|
614
1350
|
|
|
615
|
-
expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.
|
|
1351
|
+
expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.replicationStreamId);
|
|
616
1352
|
await syncRules.current_lock?.release();
|
|
617
1353
|
});
|
|
618
1354
|
|
|
1355
|
+
test.runIf(storageVersion >= 3)('does not lock when appending a sync config to an existing stream', async () => {
|
|
1356
|
+
await using factory = await storageConfig.factory();
|
|
1357
|
+
|
|
1358
|
+
const firstRules = `
|
|
1359
|
+
config:
|
|
1360
|
+
edition: 3
|
|
1361
|
+
|
|
1362
|
+
streams:
|
|
1363
|
+
by_owner:
|
|
1364
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1365
|
+
`;
|
|
1366
|
+
const secondRules = `
|
|
1367
|
+
config:
|
|
1368
|
+
edition: 3
|
|
1369
|
+
|
|
1370
|
+
streams:
|
|
1371
|
+
by_project:
|
|
1372
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1373
|
+
`;
|
|
1374
|
+
|
|
1375
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstRules, { storageVersion, lock: true }));
|
|
1376
|
+
expect(first.current_lock?.sync_rules_id).toBe(first.replicationStreamId);
|
|
1377
|
+
try {
|
|
1378
|
+
const firstStorage = factory.getInstance(first);
|
|
1379
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1380
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1381
|
+
await firstWriter.commit('1/1');
|
|
1382
|
+
|
|
1383
|
+
const second = await factory.updateSyncRules(
|
|
1384
|
+
updateSyncRulesFromYaml(secondRules, { storageVersion, lock: true })
|
|
1385
|
+
);
|
|
1386
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1387
|
+
expect(second.current_lock).toBeNull();
|
|
1388
|
+
} finally {
|
|
1389
|
+
await first.current_lock?.release();
|
|
1390
|
+
}
|
|
1391
|
+
});
|
|
1392
|
+
|
|
619
1393
|
test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
|
|
620
1394
|
await using factory = await storageConfig.factory();
|
|
621
1395
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -647,10 +1421,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
647
1421
|
await writer.commit('1/1');
|
|
648
1422
|
|
|
649
1423
|
const mongoFactory = factory as MongoBucketStorage;
|
|
650
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1424
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
651
1425
|
|
|
652
1426
|
const sourceRecordCollections = await mongoFactory.db.db
|
|
653
|
-
.listCollections({ name: new RegExp(`^source_records_${syncRules.
|
|
1427
|
+
.listCollections({ name: new RegExp(`^source_records_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
|
|
654
1428
|
.toArray();
|
|
655
1429
|
expect(sourceRecordCollections).toEqual([]);
|
|
656
1430
|
});
|
|
@@ -686,13 +1460,103 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
686
1460
|
await writer.commit('1/1');
|
|
687
1461
|
|
|
688
1462
|
const mongoFactory = factory as MongoBucketStorage;
|
|
689
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1463
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
690
1464
|
|
|
691
1465
|
await bucketStorage.clear();
|
|
692
1466
|
|
|
693
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1467
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
|
|
694
1468
|
});
|
|
695
1469
|
|
|
1470
|
+
test.runIf(storageVersion < 3)(
|
|
1471
|
+
'clear removes large v1 current_data sets without affecting other streams',
|
|
1472
|
+
async () => {
|
|
1473
|
+
await using factory = await storageConfig.factory();
|
|
1474
|
+
const syncRules = await factory.updateSyncRules(
|
|
1475
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
|
|
1476
|
+
storageVersion
|
|
1477
|
+
})
|
|
1478
|
+
);
|
|
1479
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1480
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1481
|
+
const sourceTableId = new bson.ObjectId();
|
|
1482
|
+
const documents = Array.from({ length: 10_002 }, (_, index) => ({
|
|
1483
|
+
_id: {
|
|
1484
|
+
g: syncRules.replicationStreamId,
|
|
1485
|
+
t: sourceTableId,
|
|
1486
|
+
k: index
|
|
1487
|
+
},
|
|
1488
|
+
data: new bson.Binary(),
|
|
1489
|
+
buckets: [],
|
|
1490
|
+
lookups: []
|
|
1491
|
+
}));
|
|
1492
|
+
await mongoFactory.db.current_data.insertMany([
|
|
1493
|
+
...documents,
|
|
1494
|
+
{
|
|
1495
|
+
_id: {
|
|
1496
|
+
g: syncRules.replicationStreamId + 1,
|
|
1497
|
+
t: sourceTableId,
|
|
1498
|
+
k: 0
|
|
1499
|
+
},
|
|
1500
|
+
data: new bson.Binary(),
|
|
1501
|
+
buckets: [],
|
|
1502
|
+
lookups: []
|
|
1503
|
+
}
|
|
1504
|
+
]);
|
|
1505
|
+
|
|
1506
|
+
await bucketStorage.clear();
|
|
1507
|
+
|
|
1508
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
|
|
1509
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId + 1 })).toBe(1);
|
|
1510
|
+
}
|
|
1511
|
+
);
|
|
1512
|
+
|
|
1513
|
+
test.runIf(storageVersion < 3)(
|
|
1514
|
+
'clear removes large v1 parameter index sets without affecting other streams',
|
|
1515
|
+
async () => {
|
|
1516
|
+
await using factory = await storageConfig.factory();
|
|
1517
|
+
const syncRules = await factory.updateSyncRules(
|
|
1518
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
|
|
1519
|
+
storageVersion
|
|
1520
|
+
})
|
|
1521
|
+
);
|
|
1522
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1523
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1524
|
+
const sourceTableId = new bson.ObjectId();
|
|
1525
|
+
const documents = Array.from({ length: 10_002 }, (_, index) => ({
|
|
1526
|
+
_id: BigInt(index * 2),
|
|
1527
|
+
key: {
|
|
1528
|
+
g: syncRules.replicationStreamId,
|
|
1529
|
+
t: sourceTableId,
|
|
1530
|
+
k: index
|
|
1531
|
+
},
|
|
1532
|
+
lookup: new bson.Binary(),
|
|
1533
|
+
bucket_parameters: []
|
|
1534
|
+
}));
|
|
1535
|
+
await mongoFactory.db.bucket_parameters.insertMany([
|
|
1536
|
+
...documents,
|
|
1537
|
+
{
|
|
1538
|
+
_id: 1n,
|
|
1539
|
+
key: {
|
|
1540
|
+
g: syncRules.replicationStreamId + 1,
|
|
1541
|
+
t: sourceTableId,
|
|
1542
|
+
k: 0
|
|
1543
|
+
},
|
|
1544
|
+
lookup: new bson.Binary(),
|
|
1545
|
+
bucket_parameters: []
|
|
1546
|
+
}
|
|
1547
|
+
]);
|
|
1548
|
+
|
|
1549
|
+
await bucketStorage.clear();
|
|
1550
|
+
|
|
1551
|
+
expect(await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId })).toBe(
|
|
1552
|
+
0
|
|
1553
|
+
);
|
|
1554
|
+
expect(
|
|
1555
|
+
await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId + 1 })
|
|
1556
|
+
).toBe(1);
|
|
1557
|
+
}
|
|
1558
|
+
);
|
|
1559
|
+
|
|
696
1560
|
test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
|
|
697
1561
|
await using factory = await storageConfig.factory();
|
|
698
1562
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -725,7 +1589,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
725
1589
|
await writer.commit('1/1');
|
|
726
1590
|
|
|
727
1591
|
const mongoFactory = factory as MongoBucketStorage;
|
|
728
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1592
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
729
1593
|
|
|
730
1594
|
const metricsAfter = await factory.getStorageMetrics();
|
|
731
1595
|
expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
|
|
@@ -783,7 +1647,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
783
1647
|
}
|
|
784
1648
|
);
|
|
785
1649
|
|
|
786
|
-
test.runIf(storageVersion
|
|
1650
|
+
test.runIf(storageVersion == 3)('cleans pending deletes only for tracked v3 source tables', async () => {
|
|
787
1651
|
await using factory = await storageConfig.factory();
|
|
788
1652
|
const syncRules = await factory.updateSyncRules(
|
|
789
1653
|
updateSyncRulesFromYaml(
|
|
@@ -798,21 +1662,20 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
798
1662
|
);
|
|
799
1663
|
|
|
800
1664
|
const mongoFactory = factory as MongoBucketStorage;
|
|
801
|
-
const bucketStorage = mongoFactory.getInstance(syncRules)
|
|
802
|
-
const db = bucketStorage.db;
|
|
803
|
-
await db.initializeStreamStorage(syncRules.
|
|
1665
|
+
const bucketStorage = mongoFactory.getInstance(syncRules);
|
|
1666
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1667
|
+
await db.initializeStreamStorage(syncRules.replicationStreamId);
|
|
804
1668
|
|
|
805
1669
|
const sourceTableA = new bson.ObjectId();
|
|
806
1670
|
const sourceTableB = new bson.ObjectId();
|
|
807
|
-
await db.
|
|
1671
|
+
await db.sourceTables(syncRules.replicationStreamId).insertMany([
|
|
808
1672
|
{
|
|
809
1673
|
_id: sourceTableA,
|
|
810
1674
|
connection_id: 1,
|
|
811
1675
|
relation_id: 'a',
|
|
812
1676
|
schema_name: 'public',
|
|
813
1677
|
table_name: 'table_a',
|
|
814
|
-
replica_id_columns:
|
|
815
|
-
replica_id_columns2: [],
|
|
1678
|
+
replica_id_columns: [],
|
|
816
1679
|
snapshot_done: true,
|
|
817
1680
|
snapshot_status: undefined,
|
|
818
1681
|
bucket_data_source_ids: [],
|
|
@@ -825,8 +1688,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
825
1688
|
relation_id: 'b',
|
|
826
1689
|
schema_name: 'public',
|
|
827
1690
|
table_name: 'table_b',
|
|
828
|
-
replica_id_columns:
|
|
829
|
-
replica_id_columns2: [],
|
|
1691
|
+
replica_id_columns: [],
|
|
830
1692
|
snapshot_done: true,
|
|
831
1693
|
snapshot_status: undefined,
|
|
832
1694
|
bucket_data_source_ids: [],
|
|
@@ -835,35 +1697,53 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
835
1697
|
}
|
|
836
1698
|
]);
|
|
837
1699
|
|
|
838
|
-
await db.
|
|
1700
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).insertMany([
|
|
839
1701
|
{ _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
|
|
840
1702
|
{ _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
|
|
841
1703
|
{ _id: 'active', data: null, buckets: [], lookups: [] }
|
|
842
1704
|
]);
|
|
843
1705
|
await db
|
|
844
|
-
.
|
|
1706
|
+
.sourceRecords(syncRules.replicationStreamId, sourceTableB)
|
|
845
1707
|
.insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
|
|
846
1708
|
|
|
847
|
-
const store = new SourceRecordStoreV3(
|
|
1709
|
+
const store = new SourceRecordStoreV3(
|
|
1710
|
+
db,
|
|
1711
|
+
syncRules.replicationStreamId,
|
|
1712
|
+
bucketStorage.replicationStream.syncConfigContent[0].mapping
|
|
1713
|
+
);
|
|
848
1714
|
const logger = { info() {} } as any;
|
|
849
1715
|
|
|
850
1716
|
await store.postCommitCleanup(6n, logger);
|
|
851
1717
|
|
|
852
|
-
expect(
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
expect(
|
|
856
|
-
|
|
1718
|
+
expect(
|
|
1719
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 5n })
|
|
1720
|
+
).toBe(0);
|
|
1721
|
+
expect(
|
|
1722
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 9n })
|
|
1723
|
+
).toBe(1);
|
|
1724
|
+
expect(
|
|
1725
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableB).countDocuments({ pending_delete: 12n })
|
|
1726
|
+
).toBe(1);
|
|
1727
|
+
expect(
|
|
1728
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
|
|
1729
|
+
).toBe(9n);
|
|
1730
|
+
expect(
|
|
1731
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
|
|
1732
|
+
).toBe(12n);
|
|
857
1733
|
|
|
858
1734
|
await store.postCommitCleanup(10n, logger);
|
|
859
1735
|
|
|
860
1736
|
expect(
|
|
861
|
-
await db
|
|
1737
|
+
await db
|
|
1738
|
+
.sourceRecords(syncRules.replicationStreamId, sourceTableA)
|
|
1739
|
+
.countDocuments({ pending_delete: { $exists: true } })
|
|
862
1740
|
).toBe(0);
|
|
863
1741
|
expect(
|
|
864
|
-
(await db.
|
|
1742
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
|
|
865
1743
|
).toBeUndefined();
|
|
866
|
-
expect(
|
|
1744
|
+
expect(
|
|
1745
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
|
|
1746
|
+
).toBe(12n);
|
|
867
1747
|
});
|
|
868
1748
|
}
|
|
869
1749
|
|
|
@@ -880,7 +1760,7 @@ describe('sync - mongodb', () => {
|
|
|
880
1760
|
await legacyWriter.markAllSnapshotDone('1/1');
|
|
881
1761
|
await legacyWriter.commit('1/1');
|
|
882
1762
|
|
|
883
|
-
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.
|
|
1763
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
|
|
884
1764
|
storage.SyncRuleState.ACTIVE
|
|
885
1765
|
);
|
|
886
1766
|
|
|
@@ -892,7 +1772,7 @@ describe('sync - mongodb', () => {
|
|
|
892
1772
|
await v3Writer.markAllSnapshotDone('2/1');
|
|
893
1773
|
await v3Writer.commit('2/1');
|
|
894
1774
|
|
|
895
|
-
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.
|
|
1775
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
|
|
896
1776
|
storage.SyncRuleState.STOP
|
|
897
1777
|
);
|
|
898
1778
|
});
|
|
@@ -900,6 +1780,233 @@ describe('sync - mongodb', () => {
|
|
|
900
1780
|
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
901
1781
|
describe(`storage v${storageVersion}`, () => {
|
|
902
1782
|
registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
|
|
1783
|
+
|
|
1784
|
+
describe.runIf(storageVersion == 3)('V3 read filtering boundaries', () => {
|
|
1785
|
+
async function setupFilteringTest() {
|
|
1786
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1787
|
+
const syncRules = await factory.updateSyncRules(
|
|
1788
|
+
updateSyncRulesFromYaml(
|
|
1789
|
+
`
|
|
1790
|
+
bucket_definitions:
|
|
1791
|
+
global:
|
|
1792
|
+
data:
|
|
1793
|
+
- SELECT id, description FROM test
|
|
1794
|
+
`,
|
|
1795
|
+
{ storageVersion }
|
|
1796
|
+
)
|
|
1797
|
+
);
|
|
1798
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
1799
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1800
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
1801
|
+
|
|
1802
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
1803
|
+
const definitionId = mapping.bucketSourceId(request.source);
|
|
1804
|
+
const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
|
|
1805
|
+
|
|
1806
|
+
const bucketName = request.bucket;
|
|
1807
|
+
const sourceTable = new bson.ObjectId();
|
|
1808
|
+
const bucketKey: BucketKey = {
|
|
1809
|
+
replicationStreamId: syncRules.replicationStreamId,
|
|
1810
|
+
definitionId,
|
|
1811
|
+
bucket: bucketName
|
|
1812
|
+
};
|
|
1813
|
+
|
|
1814
|
+
function makeOps(opIds: bigint[]): BucketDataDoc[] {
|
|
1815
|
+
return opIds.map((opId) => ({
|
|
1816
|
+
bucketKey,
|
|
1817
|
+
o: opId,
|
|
1818
|
+
op: 'PUT' as const,
|
|
1819
|
+
source_table: sourceTable,
|
|
1820
|
+
source_key: test_utils.rid(`row-${opId}`),
|
|
1821
|
+
table: 'items',
|
|
1822
|
+
row_id: `row-${opId}`,
|
|
1823
|
+
checksum: BigInt(opId) * 10n,
|
|
1824
|
+
data: `{"id":"row-${opId}"}`
|
|
1825
|
+
}));
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
const docA = serializeBucketData(bucketName, makeOps([10n, 20n, 30n]));
|
|
1829
|
+
const docB = serializeBucketData(bucketName, makeOps([40n, 50n, 60n]));
|
|
1830
|
+
const docC = serializeBucketData(bucketName, makeOps([70n, 80n, 90n]));
|
|
1831
|
+
|
|
1832
|
+
await collection.insertMany([docA, docB, docC]);
|
|
1833
|
+
|
|
1834
|
+
return { factory, syncRules, bucketStorage, bucketName };
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
|
|
1838
|
+
const { syncRules, bucketStorage } = await setupFilteringTest();
|
|
1839
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
|
|
1840
|
+
const batch = await test_utils.getBatchArray(
|
|
1841
|
+
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(BigInt(checkpoint)), [request])
|
|
1842
|
+
);
|
|
1843
|
+
const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
|
|
1844
|
+
return ops;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
test('case 1: start=5, checkpoint=95 → all ops', async () => {
|
|
1848
|
+
const ops = await getFilteredOps(5, 95);
|
|
1849
|
+
expect(ops).toEqual([10n, 20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
|
|
1850
|
+
});
|
|
1851
|
+
|
|
1852
|
+
test('case 2: start=10, checkpoint=90 → ops in (10,90]', async () => {
|
|
1853
|
+
const ops = await getFilteredOps(10, 90);
|
|
1854
|
+
expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
|
|
1855
|
+
});
|
|
1856
|
+
|
|
1857
|
+
test('case 3: start=15, checkpoint=85 → partial doc boundaries', async () => {
|
|
1858
|
+
const ops = await getFilteredOps(15, 85);
|
|
1859
|
+
expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n]);
|
|
1860
|
+
});
|
|
1861
|
+
|
|
1862
|
+
test('case 4: start=25, checkpoint=55 → spans two docs', async () => {
|
|
1863
|
+
const ops = await getFilteredOps(25, 55);
|
|
1864
|
+
expect(ops).toEqual([30n, 40n, 50n]);
|
|
1865
|
+
});
|
|
1866
|
+
|
|
1867
|
+
test('case 5: start=35, checkpoint=45 → single op within doc', async () => {
|
|
1868
|
+
const ops = await getFilteredOps(35, 45);
|
|
1869
|
+
expect(ops).toEqual([40n]);
|
|
1870
|
+
});
|
|
1871
|
+
|
|
1872
|
+
test('case 6: start=35, checkpoint=65 → full doc B', async () => {
|
|
1873
|
+
const ops = await getFilteredOps(35, 65);
|
|
1874
|
+
expect(ops).toEqual([40n, 50n, 60n]);
|
|
1875
|
+
});
|
|
1876
|
+
|
|
1877
|
+
test('case 7: start=25, checkpoint=35 → single op from doc A', async () => {
|
|
1878
|
+
const ops = await getFilteredOps(25, 35);
|
|
1879
|
+
expect(ops).toEqual([30n]);
|
|
1880
|
+
});
|
|
1881
|
+
|
|
1882
|
+
test('case 8: start=30, checkpoint=40 → op at checkpoint from next doc', async () => {
|
|
1883
|
+
const ops = await getFilteredOps(30, 40);
|
|
1884
|
+
expect(ops).toEqual([40n]);
|
|
1885
|
+
});
|
|
1886
|
+
|
|
1887
|
+
test('case 9: start=100, checkpoint=200 → beyond all docs', async () => {
|
|
1888
|
+
const ops = await getFilteredOps(100, 200);
|
|
1889
|
+
expect(ops).toEqual([]);
|
|
1890
|
+
});
|
|
1891
|
+
|
|
1892
|
+
test('case 10: start=0, checkpoint=5 → before all docs', async () => {
|
|
1893
|
+
const ops = await getFilteredOps(0, 5);
|
|
1894
|
+
expect(ops).toEqual([]);
|
|
1895
|
+
});
|
|
1896
|
+
|
|
1897
|
+
test('case 11: start=50, checkpoint=50 → zero-width range', async () => {
|
|
1898
|
+
const ops = await getFilteredOps(50, 50);
|
|
1899
|
+
expect(ops).toEqual([]);
|
|
1900
|
+
});
|
|
1901
|
+
|
|
1902
|
+
test('case 12: start=45, checkpoint=50 → op at checkpoint boundary', async () => {
|
|
1903
|
+
const ops = await getFilteredOps(45, 50);
|
|
1904
|
+
expect(ops).toEqual([50n]);
|
|
1905
|
+
});
|
|
1906
|
+
|
|
1907
|
+
test('case 13: start=50, checkpoint=55 → no ops strictly after start', async () => {
|
|
1908
|
+
const ops = await getFilteredOps(50, 55);
|
|
1909
|
+
expect(ops).toEqual([]);
|
|
1910
|
+
});
|
|
1911
|
+
|
|
1912
|
+
test('all-filtered first batch still returns data behind the batch boundary', async () => {
|
|
1913
|
+
// Documents straddling the requested (start, end] window are matched by the
|
|
1914
|
+
// query, but contribute no rows after filtering. If an entire server batch
|
|
1915
|
+
// (~101 documents) consists of such straddlers, the remaining documents in
|
|
1916
|
+
// the cursor must still be reachable. Storage reports the straddler buckets
|
|
1917
|
+
// as complete via empty chunks, and the caller re-requests the rest.
|
|
1918
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1919
|
+
const syncRules = await factory.updateSyncRules(
|
|
1920
|
+
updateSyncRulesFromYaml(
|
|
1921
|
+
`
|
|
1922
|
+
bucket_definitions:
|
|
1923
|
+
by_user:
|
|
1924
|
+
parameters: select request.user_id() as user_id
|
|
1925
|
+
data: [select * from test where owner_id = bucket.user_id]
|
|
1926
|
+
`,
|
|
1927
|
+
{ storageVersion }
|
|
1928
|
+
)
|
|
1929
|
+
);
|
|
1930
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3;
|
|
1931
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1932
|
+
|
|
1933
|
+
const start = 5n;
|
|
1934
|
+
const end = 50n;
|
|
1935
|
+
|
|
1936
|
+
// 150 buckets sorted before the data bucket, each with a single document
|
|
1937
|
+
// containing ops at 1 and 100: matched (_id.o=100 > start, min_op=1 <= end),
|
|
1938
|
+
// but no op in (5, 50].
|
|
1939
|
+
const straddlerNames = Array.from({ length: 150 }, (_, i) => `b${`${i}`.padStart(3, '0')}`);
|
|
1940
|
+
const requests = [...straddlerNames, 'zzz'].map((id) =>
|
|
1941
|
+
bucketRequest(syncRules.syncConfigContent[0], `by_user["${id}"]`, start)
|
|
1942
|
+
);
|
|
1943
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
1944
|
+
const definitionId = mapping.bucketSourceId(requests[0].source);
|
|
1945
|
+
const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
|
|
1946
|
+
const sourceTable = new bson.ObjectId();
|
|
1947
|
+
|
|
1948
|
+
function makeOps(bucket: string, opIds: bigint[]): BucketDataDoc[] {
|
|
1949
|
+
const bucketKey: BucketKey = {
|
|
1950
|
+
replicationStreamId: syncRules.replicationStreamId,
|
|
1951
|
+
definitionId,
|
|
1952
|
+
bucket
|
|
1953
|
+
};
|
|
1954
|
+
return opIds.map((opId) => ({
|
|
1955
|
+
bucketKey,
|
|
1956
|
+
o: opId,
|
|
1957
|
+
op: 'PUT' as const,
|
|
1958
|
+
source_table: sourceTable,
|
|
1959
|
+
source_key: test_utils.rid(`row-${opId}`),
|
|
1960
|
+
table: 'test',
|
|
1961
|
+
row_id: `row-${opId}`,
|
|
1962
|
+
checksum: BigInt(opId) * 10n,
|
|
1963
|
+
data: `{"id":"row-${opId}"}`
|
|
1964
|
+
}));
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
const straddlerDocs = requests
|
|
1968
|
+
.slice(0, -1)
|
|
1969
|
+
.map((request) => serializeBucketData(request.bucket, makeOps(request.bucket, [1n, 100n])));
|
|
1970
|
+
const dataBucket = requests[requests.length - 1].bucket;
|
|
1971
|
+
const dataDoc = serializeBucketData(dataBucket, makeOps(dataBucket, [10n]));
|
|
1972
|
+
await collection.insertMany([...straddlerDocs, dataDoc]);
|
|
1973
|
+
|
|
1974
|
+
// Emulate the caller loop in sync.ts / BucketChecksumState: advance bucket
|
|
1975
|
+
// positions from each chunk, drop completed buckets, and re-request while
|
|
1976
|
+
// any chunk reported has_more.
|
|
1977
|
+
const positions = new Map(requests.map((request) => [request.bucket, request.start]));
|
|
1978
|
+
const pending = new Set(positions.keys());
|
|
1979
|
+
const receivedOps: bigint[] = [];
|
|
1980
|
+
let rounds = 0;
|
|
1981
|
+
|
|
1982
|
+
while (rounds < 10) {
|
|
1983
|
+
rounds++;
|
|
1984
|
+
const roundRequests = requests
|
|
1985
|
+
.filter((request) => pending.has(request.bucket))
|
|
1986
|
+
.map((request) => ({ ...request, start: positions.get(request.bucket)! }));
|
|
1987
|
+
const batch = await test_utils.getBatchArray(
|
|
1988
|
+
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(end), roundRequests)
|
|
1989
|
+
);
|
|
1990
|
+
let anyHasMore = false;
|
|
1991
|
+
for (const { chunkData } of batch) {
|
|
1992
|
+
positions.set(chunkData.bucket, BigInt(chunkData.next_after));
|
|
1993
|
+
if (chunkData.has_more) {
|
|
1994
|
+
anyHasMore = true;
|
|
1995
|
+
} else {
|
|
1996
|
+
pending.delete(chunkData.bucket);
|
|
1997
|
+
}
|
|
1998
|
+
receivedOps.push(...chunkData.data.map((entry) => BigInt(entry.op_id)));
|
|
1999
|
+
}
|
|
2000
|
+
if (!anyHasMore) {
|
|
2001
|
+
break;
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
// The op behind the all-straddler first batch must be returned.
|
|
2006
|
+
expect(receivedOps).toEqual([10n]);
|
|
2007
|
+
expect(rounds).toBeLessThan(10);
|
|
2008
|
+
});
|
|
2009
|
+
});
|
|
903
2010
|
});
|
|
904
2011
|
}
|
|
905
2012
|
});
|