@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/storage/MongoBucketStorage.d.ts +35 -15
- package/dist/storage/MongoBucketStorage.js +309 -86
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/CheckpointState.d.ts +20 -0
- package/dist/storage/implementation/CheckpointState.js +31 -0
- package/dist/storage/implementation/CheckpointState.js.map +1 -0
- package/dist/storage/implementation/MongoBucketBatch.d.ts +69 -39
- package/dist/storage/implementation/MongoBucketBatch.js +104 -313
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +11 -9
- package/dist/storage/implementation/MongoChecksums.js +3 -111
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +3 -1
- package/dist/storage/implementation/MongoCompactor.js +18 -9
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
- package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +121 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +5 -2
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +50 -21
- package/dist/storage/implementation/MongoSyncBucketStorage.js +62 -199
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +9 -5
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
- package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -3
- package/dist/storage/implementation/common/PersistedBatch.js +5 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +10 -2
- package/dist/storage/implementation/db.js +9 -14
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +25 -50
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +23 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +403 -2
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +22 -15
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +100 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +21 -1
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +21 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +459 -3
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +20 -3
- package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +50 -16
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +292 -75
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +2 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +16 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
- package/dist/storage/implementation/v3/bucket-format.js +58 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +28 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +111 -10
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +2 -2
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +5 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +1 -1
- package/dist/utils/util.js +3 -2
- package/dist/utils/util.js.map +1 -1
- package/package.json +9 -19
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
- package/src/storage/MongoBucketStorage.ts +464 -95
- package/src/storage/implementation/CheckpointState.ts +59 -0
- package/src/storage/implementation/MongoBucketBatch.ts +159 -405
- package/src/storage/implementation/MongoChecksums.ts +7 -138
- package/src/storage/implementation/MongoCompactor.ts +20 -10
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +152 -0
- package/src/storage/implementation/MongoStorageProvider.ts +5 -2
- package/src/storage/implementation/MongoSyncBucketStorage.ts +89 -250
- package/src/storage/implementation/MongoSyncRulesLock.ts +14 -5
- package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
- package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
- package/src/storage/implementation/common/PersistedBatch.ts +8 -4
- package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +16 -21
- package/src/storage/implementation/models.ts +28 -61
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +503 -5
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
- package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +150 -49
- package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +25 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +611 -4
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
- package/src/storage/implementation/v3/MongoCompactorV3.ts +647 -30
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +397 -111
- package/src/storage/implementation/v3/PersistedBatchV3.ts +63 -38
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +48 -38
- package/src/storage/implementation/v3/bucket-format.ts +70 -0
- package/src/storage/implementation/v3/chunking.ts +38 -0
- package/src/storage/implementation/v3/models.ts +150 -44
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/utils/test-utils.ts +6 -3
- package/src/utils/util.ts +3 -2
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
- package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
- package/test/src/storage.test.ts +11 -11
- package/test/src/storage_compacting.test.ts +1956 -6
- package/test/src/storage_sync.test.ts +1429 -42
- package/test/src/storeCurrentData.test.ts +221 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -17
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -64
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -15
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -42
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -5
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -72
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -76
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -56
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -12
|
@@ -1,16 +1,119 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
|
|
2
|
+
import {
|
|
3
|
+
deserializeParameterLookup,
|
|
4
|
+
JwtPayload,
|
|
5
|
+
SingleSyncConfigBucketDefinitionMapping,
|
|
6
|
+
storage,
|
|
7
|
+
updateSyncRulesFromYaml
|
|
8
|
+
} from '@powersync/service-core';
|
|
2
9
|
import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
|
|
3
|
-
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
10
|
+
import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
4
11
|
import * as bson from 'bson';
|
|
5
12
|
import { describe, expect, test } from 'vitest';
|
|
6
13
|
import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
|
|
14
|
+
import { MongoParsedSyncConfigSet } from '../../src/storage/implementation/MongoParsedSyncConfigSet.js';
|
|
15
|
+
import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementation/MongoPersistedSyncConfigContent.js';
|
|
16
|
+
import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
|
|
7
17
|
import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
|
|
8
|
-
import {
|
|
18
|
+
import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
|
|
9
19
|
import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
|
|
10
20
|
import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
11
|
-
import {
|
|
21
|
+
import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
|
|
22
|
+
import {
|
|
23
|
+
CurrentBucketV3,
|
|
24
|
+
ReplicationStreamDocumentV3,
|
|
25
|
+
SyncConfigDefinition
|
|
26
|
+
} from '../../src/storage/implementation/v3/models.js';
|
|
12
27
|
import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
|
|
13
28
|
|
|
29
|
+
const MINIMAL_SYNC_RULES = `
|
|
30
|
+
bucket_definitions:
|
|
31
|
+
global:
|
|
32
|
+
data:
|
|
33
|
+
- SELECT id FROM test
|
|
34
|
+
`;
|
|
35
|
+
|
|
36
|
+
function sourceDescriptor(
|
|
37
|
+
name: string,
|
|
38
|
+
options: {
|
|
39
|
+
objectId?: string;
|
|
40
|
+
replicaIdColumns?: string[];
|
|
41
|
+
} = {}
|
|
42
|
+
): storage.SourceEntityDescriptor {
|
|
43
|
+
return {
|
|
44
|
+
connectionTag: storage.SourceTable.DEFAULT_TAG,
|
|
45
|
+
objectId: options.objectId ?? name,
|
|
46
|
+
schema: 'public',
|
|
47
|
+
name,
|
|
48
|
+
replicaIdColumns: (options.replicaIdColumns ?? ['id']).map((column) => ({
|
|
49
|
+
name: column,
|
|
50
|
+
type: 'VARCHAR',
|
|
51
|
+
typeId: 25
|
|
52
|
+
}))
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function objectIdGenerator(id: string) {
|
|
57
|
+
let used = false;
|
|
58
|
+
return () => {
|
|
59
|
+
if (used) {
|
|
60
|
+
throw new Error(`Can only generate a single id using ${id}`);
|
|
61
|
+
}
|
|
62
|
+
used = true;
|
|
63
|
+
return new bson.ObjectId(id);
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Build a standalone parsed sync config set for the given yaml, with its own mapping.
|
|
69
|
+
*
|
|
70
|
+
* This keeps the sync rules and the bucket definition mapping paired, as required by
|
|
71
|
+
* `ResolveTablesOptions.parsedSyncConfig`.
|
|
72
|
+
*/
|
|
73
|
+
function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
|
|
74
|
+
const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
|
|
75
|
+
expect(parsed.errors).toEqual([]);
|
|
76
|
+
return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
|
|
77
|
+
{
|
|
78
|
+
syncConfigId: new bson.ObjectId().toHexString(),
|
|
79
|
+
syncConfig: parsed,
|
|
80
|
+
mapping: SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(parsed)
|
|
81
|
+
}
|
|
82
|
+
]);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory, replicationStreamId: number) {
|
|
86
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
87
|
+
const doc = (await mongoFactory.db.sync_rules.findOne({
|
|
88
|
+
_id: replicationStreamId
|
|
89
|
+
})) as ReplicationStreamDocumentV3 | null;
|
|
90
|
+
if (doc == null) {
|
|
91
|
+
return [];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const db = mongoFactory.db.versioned(getMongoStorageConfig(doc.storage_version)) as VersionedPowerSyncMongoV3;
|
|
95
|
+
const syncConfigDocs = await db.syncConfigDefinitions
|
|
96
|
+
.find({
|
|
97
|
+
_id: { $in: doc.sync_configs.map((config) => config._id) }
|
|
98
|
+
})
|
|
99
|
+
.toArray();
|
|
100
|
+
|
|
101
|
+
return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Get a MongoDB V3 storage bucket definition id, from a bucket name.
|
|
106
|
+
*
|
|
107
|
+
* This relies on internals of bucket naming, but helps for simplifying tests.
|
|
108
|
+
*/
|
|
109
|
+
function bucketDefinitionId(bucket: string) {
|
|
110
|
+
const match = bucket.match(/^\w+\|\w+\.\w+\.(\w+)\[/);
|
|
111
|
+
if (match == null) {
|
|
112
|
+
throw new Error(`Expected versioned bucket name, got ${bucket}`);
|
|
113
|
+
}
|
|
114
|
+
return parseInt(match[1], 16);
|
|
115
|
+
}
|
|
116
|
+
|
|
14
117
|
function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
|
|
15
118
|
register.registerSyncTests(storageConfig.factory, {
|
|
16
119
|
storageVersion,
|
|
@@ -34,6 +137,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
34
137
|
)
|
|
35
138
|
);
|
|
36
139
|
const bucketStorage = factory.getInstance(syncRules);
|
|
140
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
37
141
|
|
|
38
142
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
39
143
|
|
|
@@ -88,7 +192,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
88
192
|
|
|
89
193
|
const options: storage.BucketDataBatchOptions = {};
|
|
90
194
|
const batch1 = await test_utils.fromAsync(
|
|
91
|
-
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(
|
|
195
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)], options)
|
|
92
196
|
);
|
|
93
197
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
94
198
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
@@ -103,7 +207,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
103
207
|
const batch2 = await test_utils.fromAsync(
|
|
104
208
|
bucketStorage.getBucketDataBatch(
|
|
105
209
|
checkpoint,
|
|
106
|
-
[bucketRequest(
|
|
210
|
+
[bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
|
|
107
211
|
options
|
|
108
212
|
)
|
|
109
213
|
);
|
|
@@ -119,7 +223,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
119
223
|
const batch3 = await test_utils.fromAsync(
|
|
120
224
|
bucketStorage.getBucketDataBatch(
|
|
121
225
|
checkpoint,
|
|
122
|
-
[bucketRequest(
|
|
226
|
+
[bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
|
|
123
227
|
options
|
|
124
228
|
)
|
|
125
229
|
);
|
|
@@ -141,7 +245,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
141
245
|
await Promise.all(
|
|
142
246
|
(
|
|
143
247
|
await mongoFactory.db.db
|
|
144
|
-
.listCollections(
|
|
248
|
+
.listCollections(
|
|
249
|
+
{ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) },
|
|
250
|
+
{ nameOnly: true }
|
|
251
|
+
)
|
|
145
252
|
.toArray()
|
|
146
253
|
).map((collection: { name: string }) =>
|
|
147
254
|
mongoFactory.db.db
|
|
@@ -157,6 +264,386 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
157
264
|
expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]);
|
|
158
265
|
});
|
|
159
266
|
|
|
267
|
+
test('resolveTables populates matching data and parameter sources', async () => {
|
|
268
|
+
await using factory = await storageConfig.factory();
|
|
269
|
+
const syncRules = await factory.updateSyncRules(
|
|
270
|
+
updateSyncRulesFromYaml(
|
|
271
|
+
`
|
|
272
|
+
bucket_definitions:
|
|
273
|
+
by_owner:
|
|
274
|
+
parameters:
|
|
275
|
+
- SELECT owner_id FROM test WHERE id = token_parameters.test_id
|
|
276
|
+
data:
|
|
277
|
+
- SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id
|
|
278
|
+
`,
|
|
279
|
+
{ storageVersion }
|
|
280
|
+
)
|
|
281
|
+
);
|
|
282
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
283
|
+
|
|
284
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
285
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
286
|
+
|
|
287
|
+
expect(sourceTable.bucketDataSources).toHaveLength(1);
|
|
288
|
+
expect(sourceTable.parameterLookupSources).toHaveLength(1);
|
|
289
|
+
expect(sourceTable.syncData).toBe(true);
|
|
290
|
+
expect(sourceTable.syncParameters).toBe(true);
|
|
291
|
+
expect(sourceTable.syncEvent).toBe(false);
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
test('resolveTables drops old table when table name changes for the same objectId', async () => {
|
|
295
|
+
await using factory = await storageConfig.factory();
|
|
296
|
+
const syncRules = await factory.updateSyncRules(
|
|
297
|
+
updateSyncRulesFromYaml(
|
|
298
|
+
`
|
|
299
|
+
bucket_definitions:
|
|
300
|
+
global:
|
|
301
|
+
data:
|
|
302
|
+
- SELECT id FROM "%"
|
|
303
|
+
`,
|
|
304
|
+
{ storageVersion }
|
|
305
|
+
)
|
|
306
|
+
);
|
|
307
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
308
|
+
|
|
309
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
310
|
+
const before = await writer.resolveTables({
|
|
311
|
+
connection_id: 1,
|
|
312
|
+
source: sourceDescriptor('orders', { objectId: 'orders-relation' }),
|
|
313
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38342')
|
|
314
|
+
});
|
|
315
|
+
const after = await writer.resolveTables({
|
|
316
|
+
connection_id: 1,
|
|
317
|
+
source: sourceDescriptor('renamed_orders', { objectId: 'orders-relation' }),
|
|
318
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38343')
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
expect(after.tables).toHaveLength(1);
|
|
322
|
+
expect(after.tables[0].id).not.toEqual(before.tables[0].id);
|
|
323
|
+
expect(after.tables[0].bucketDataSources).toHaveLength(1);
|
|
324
|
+
expect(after.tables[0].parameterLookupSources).toHaveLength(0);
|
|
325
|
+
expect(after.dropTables.map((table) => ({ id: table.id, name: table.name }))).toEqual([
|
|
326
|
+
{ id: before.tables[0].id, name: 'orders' }
|
|
327
|
+
]);
|
|
328
|
+
expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
|
|
329
|
+
expect(after.dropTables[0].parameterLookupSources).toHaveLength(0);
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
test('resolveTables drops old table when objectId changes for the same table name', async () => {
|
|
333
|
+
await using factory = await storageConfig.factory();
|
|
334
|
+
const syncRules = await factory.updateSyncRules(
|
|
335
|
+
updateSyncRulesFromYaml(
|
|
336
|
+
`
|
|
337
|
+
bucket_definitions:
|
|
338
|
+
global:
|
|
339
|
+
data:
|
|
340
|
+
- SELECT id FROM "%"
|
|
341
|
+
`,
|
|
342
|
+
{ storageVersion }
|
|
343
|
+
)
|
|
344
|
+
);
|
|
345
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
346
|
+
|
|
347
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
348
|
+
const before = await writer.resolveTables({
|
|
349
|
+
connection_id: 1,
|
|
350
|
+
source: sourceDescriptor('accounts', { objectId: 'accounts-relation-old' }),
|
|
351
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38344')
|
|
352
|
+
});
|
|
353
|
+
const after = await writer.resolveTables({
|
|
354
|
+
connection_id: 1,
|
|
355
|
+
source: sourceDescriptor('accounts', { objectId: 'accounts-relation-new' }),
|
|
356
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38345')
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
expect(after.tables).toHaveLength(1);
|
|
360
|
+
expect(after.tables[0].id).not.toEqual(before.tables[0].id);
|
|
361
|
+
expect(after.tables[0].bucketDataSources).toHaveLength(1);
|
|
362
|
+
expect(after.dropTables.map((table) => ({ id: table.id, objectId: table.objectId }))).toEqual([
|
|
363
|
+
{ id: before.tables[0].id, objectId: 'accounts-relation-old' }
|
|
364
|
+
]);
|
|
365
|
+
expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
test('resolveTables drops old table when replica id columns change', async () => {
|
|
369
|
+
await using factory = await storageConfig.factory();
|
|
370
|
+
const syncRules = await factory.updateSyncRules(
|
|
371
|
+
updateSyncRulesFromYaml(
|
|
372
|
+
`
|
|
373
|
+
bucket_definitions:
|
|
374
|
+
global:
|
|
375
|
+
data:
|
|
376
|
+
- SELECT id FROM "%"
|
|
377
|
+
`,
|
|
378
|
+
{ storageVersion }
|
|
379
|
+
)
|
|
380
|
+
);
|
|
381
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
382
|
+
|
|
383
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
384
|
+
const before = await writer.resolveTables({
|
|
385
|
+
connection_id: 1,
|
|
386
|
+
source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['id'] }),
|
|
387
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38346')
|
|
388
|
+
});
|
|
389
|
+
const after = await writer.resolveTables({
|
|
390
|
+
connection_id: 1,
|
|
391
|
+
source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['tenant_id', 'id'] }),
|
|
392
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38347')
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
expect(after.tables).toHaveLength(1);
|
|
396
|
+
expect(after.tables[0].id).not.toEqual(before.tables[0].id);
|
|
397
|
+
expect(after.tables[0].replicaIdColumns.map((column) => column.name)).toEqual(['tenant_id', 'id']);
|
|
398
|
+
expect(after.tables[0].bucketDataSources).toHaveLength(1);
|
|
399
|
+
expect(
|
|
400
|
+
after.dropTables.map((table) => ({ id: table.id, columns: table.replicaIdColumns.map((c) => c.name) }))
|
|
401
|
+
).toEqual([{ id: before.tables[0].id, columns: ['id'] }]);
|
|
402
|
+
expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
test.runIf(storageVersion >= 3)(
|
|
406
|
+
'resolveTables resolves v3 event-only tables without source memberships',
|
|
407
|
+
async () => {
|
|
408
|
+
await using factory = await storageConfig.factory();
|
|
409
|
+
const syncRules = await factory.updateSyncRules(
|
|
410
|
+
updateSyncRulesFromYaml(
|
|
411
|
+
`
|
|
412
|
+
bucket_definitions:
|
|
413
|
+
by_owner:
|
|
414
|
+
data:
|
|
415
|
+
- SELECT id FROM users
|
|
416
|
+
|
|
417
|
+
event_definitions:
|
|
418
|
+
write_checkpoints:
|
|
419
|
+
payloads:
|
|
420
|
+
- SELECT user_id, checkpoint FROM checkpoints
|
|
421
|
+
`,
|
|
422
|
+
{ storageVersion }
|
|
423
|
+
)
|
|
424
|
+
);
|
|
425
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
426
|
+
|
|
427
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
428
|
+
const resolved = await writer.resolveTables({
|
|
429
|
+
connection_id: 1,
|
|
430
|
+
source: {
|
|
431
|
+
connectionTag: storage.SourceTable.DEFAULT_TAG,
|
|
432
|
+
objectId: 'checkpoints',
|
|
433
|
+
schema: 'public',
|
|
434
|
+
name: 'checkpoints',
|
|
435
|
+
replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }]
|
|
436
|
+
},
|
|
437
|
+
idGenerator: () => new bson.ObjectId('6544e3899293153fa7b38341')
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
expect(resolved.tables).toHaveLength(1);
|
|
441
|
+
expect(resolved.dropTables).toHaveLength(0);
|
|
442
|
+
expect(resolved.tables[0].bucketDataSources).toEqual([]);
|
|
443
|
+
expect(resolved.tables[0].parameterLookupSources).toEqual([]);
|
|
444
|
+
expect(resolved.tables[0].syncData).toBe(false);
|
|
445
|
+
expect(resolved.tables[0].syncParameters).toBe(false);
|
|
446
|
+
expect(resolved.tables[0].syncEvent).toBe(true);
|
|
447
|
+
}
|
|
448
|
+
);
|
|
449
|
+
|
|
450
|
+
test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions', async () => {
|
|
451
|
+
// Tests the behavior of resolveTables when bucket data sources and parameter index creators are added.
|
|
452
|
+
// These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
|
|
453
|
+
// This just tests the specific resolveTables behavior.
|
|
454
|
+
|
|
455
|
+
// The same tests should work with sync streams, but legacy bucket_definitions make it easy
|
|
456
|
+
// to see the distinction between the parameter index queries and the data sources.
|
|
457
|
+
const fullRulesYaml = `
|
|
458
|
+
bucket_definitions:
|
|
459
|
+
by_owner:
|
|
460
|
+
parameters:
|
|
461
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
462
|
+
data:
|
|
463
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
464
|
+
`;
|
|
465
|
+
const dataOnlyRulesYaml = `
|
|
466
|
+
bucket_definitions:
|
|
467
|
+
by_owner:
|
|
468
|
+
parameters:
|
|
469
|
+
- SELECT token_parameters.owner_id as owner_id
|
|
470
|
+
data:
|
|
471
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
472
|
+
`;
|
|
473
|
+
const parameterOnlyRulesYaml = `
|
|
474
|
+
bucket_definitions:
|
|
475
|
+
by_owner:
|
|
476
|
+
parameters:
|
|
477
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
478
|
+
data: []
|
|
479
|
+
`;
|
|
480
|
+
const eventOnlyRulesYaml = `
|
|
481
|
+
bucket_definitions: {}
|
|
482
|
+
|
|
483
|
+
event_definitions:
|
|
484
|
+
write_checkpoints:
|
|
485
|
+
payloads:
|
|
486
|
+
- SELECT id, owner_id FROM memberships
|
|
487
|
+
`;
|
|
488
|
+
|
|
489
|
+
await using factory = await storageConfig.factory();
|
|
490
|
+
// This does not quite match what actual API usage would look like.
|
|
491
|
+
// Here we're persisting one sync config, then resolving tables with others.
|
|
492
|
+
// We're also using the default hydration state for them all.
|
|
493
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
|
|
494
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
495
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
496
|
+
const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
|
|
497
|
+
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
|
|
498
|
+
const parameterOnlyRules = parsedSyncConfigSetFor(parameterOnlyRulesYaml, storageVersion);
|
|
499
|
+
const eventOnlyRules = parsedSyncConfigSetFor(eventOnlyRulesYaml, storageVersion);
|
|
500
|
+
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
501
|
+
const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
|
|
502
|
+
const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
|
|
503
|
+
|
|
504
|
+
const dataOnly = await writer.resolveTables({
|
|
505
|
+
connection_id: 1,
|
|
506
|
+
source,
|
|
507
|
+
idGenerator: () => dataOnlyTableId,
|
|
508
|
+
parsedSyncConfig: dataOnlyRules
|
|
509
|
+
});
|
|
510
|
+
expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
511
|
+
expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
512
|
+
expect(dataOnly.tables[0].bucketDataSources).toHaveLength(1);
|
|
513
|
+
expect(dataOnly.tables[0].parameterLookupSources).toHaveLength(0);
|
|
514
|
+
|
|
515
|
+
const addedParameter = await writer.resolveTables({
|
|
516
|
+
connection_id: 1,
|
|
517
|
+
source,
|
|
518
|
+
idGenerator: () => addedParameterTableId,
|
|
519
|
+
parsedSyncConfig: fullRules
|
|
520
|
+
});
|
|
521
|
+
// Adding a definition always creates a new SourceTable
|
|
522
|
+
expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
|
|
523
|
+
expect(addedParameter.tables.map((table) => table.bucketDataSources.length).sort()).toEqual([0, 1]);
|
|
524
|
+
expect(addedParameter.tables.map((table) => table.parameterLookupSources.length).sort()).toEqual([0, 1]);
|
|
525
|
+
expect(addedParameter.dropTables.map((table) => table.id)).toEqual([]);
|
|
526
|
+
|
|
527
|
+
const removedParameter = await writer.resolveTables({
|
|
528
|
+
connection_id: 1,
|
|
529
|
+
source,
|
|
530
|
+
idGenerator: () => {
|
|
531
|
+
throw new Error('data-only resolve should reuse existing v3 source table');
|
|
532
|
+
},
|
|
533
|
+
parsedSyncConfig: dataOnlyRules
|
|
534
|
+
});
|
|
535
|
+
expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
536
|
+
// Same-identity tables that no longer cover desired memberships are left in place.
|
|
537
|
+
expect(removedParameter.dropTables.map((table) => table.id)).toEqual([]);
|
|
538
|
+
expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
|
|
539
|
+
expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
|
|
540
|
+
|
|
541
|
+
const removedData = await writer.resolveTables({
|
|
542
|
+
connection_id: 1,
|
|
543
|
+
source,
|
|
544
|
+
idGenerator: () => {
|
|
545
|
+
throw new Error('parameter-only resolve should reuse existing v3 source table');
|
|
546
|
+
},
|
|
547
|
+
parsedSyncConfig: parameterOnlyRules
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
// This goes from dataOnlyRules -> parameterOnlyRules, which removes one definition but
|
|
551
|
+
// reuses the existing parameter-only source table.
|
|
552
|
+
expect(removedData.tables.map((table) => table.id)).toEqual([addedParameterTableId]);
|
|
553
|
+
expect(removedData.dropTables.map((table) => table.id)).toEqual([]);
|
|
554
|
+
expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
|
|
555
|
+
expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
|
|
556
|
+
|
|
557
|
+
const eventOnly = await writer.resolveTables({
|
|
558
|
+
connection_id: 1,
|
|
559
|
+
source,
|
|
560
|
+
idGenerator: () => {
|
|
561
|
+
throw new Error('resolve should reuse existing v3 source table');
|
|
562
|
+
},
|
|
563
|
+
parsedSyncConfig: eventOnlyRules
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
// Event-only table can re-use any existing table.
|
|
567
|
+
expect(eventOnly.tables).toHaveLength(1);
|
|
568
|
+
expect([dataOnlyTableId.toString(), addedParameterTableId.toString()]).toContain(eventOnly.tables[0].id.toString());
|
|
569
|
+
expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
570
|
+
expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
|
|
571
|
+
expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
|
|
572
|
+
expect(eventOnly.tables[0].syncData).toBe(false);
|
|
573
|
+
expect(eventOnly.tables[0].syncParameters).toBe(false);
|
|
574
|
+
expect(eventOnly.tables[0].syncEvent).toBe(true);
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
test.runIf(storageVersion >= 3)(
|
|
578
|
+
'resolveTables designates a single event carrier across split source tables',
|
|
579
|
+
async () => {
|
|
580
|
+
// When memberships are split over multiple SourceTables for the same ref, a row change
|
|
581
|
+
// is saved once per table. Only one table may fire events, otherwise the same event
|
|
582
|
+
// would fire once per table for every row change.
|
|
583
|
+
const dataOnlyEventYaml = `
|
|
584
|
+
bucket_definitions:
|
|
585
|
+
by_owner:
|
|
586
|
+
parameters:
|
|
587
|
+
- SELECT token_parameters.owner_id as owner_id
|
|
588
|
+
data:
|
|
589
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
590
|
+
|
|
591
|
+
event_definitions:
|
|
592
|
+
write_checkpoints:
|
|
593
|
+
payloads:
|
|
594
|
+
- SELECT id, owner_id FROM memberships
|
|
595
|
+
`;
|
|
596
|
+
const fullEventYaml = `
|
|
597
|
+
bucket_definitions:
|
|
598
|
+
by_owner:
|
|
599
|
+
parameters:
|
|
600
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
601
|
+
data:
|
|
602
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
603
|
+
|
|
604
|
+
event_definitions:
|
|
605
|
+
write_checkpoints:
|
|
606
|
+
payloads:
|
|
607
|
+
- SELECT id, owner_id FROM memberships
|
|
608
|
+
`;
|
|
609
|
+
|
|
610
|
+
await using factory = await storageConfig.factory();
|
|
611
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
|
|
612
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
613
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
614
|
+
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
|
|
615
|
+
const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
|
|
616
|
+
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
617
|
+
|
|
618
|
+
const initial = await writer.resolveTables({
|
|
619
|
+
connection_id: 1,
|
|
620
|
+
source,
|
|
621
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
|
|
622
|
+
parsedSyncConfig: dataOnlyRules
|
|
623
|
+
});
|
|
624
|
+
expect(initial.tables).toHaveLength(1);
|
|
625
|
+
expect(initial.tables[0].syncEvent).toBe(true);
|
|
626
|
+
|
|
627
|
+
// Adding the table-based parameter lookup creates a second SourceTable for the same ref.
|
|
628
|
+
const split = await writer.resolveTables({
|
|
629
|
+
connection_id: 1,
|
|
630
|
+
source,
|
|
631
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
|
|
632
|
+
parsedSyncConfig: fullRules
|
|
633
|
+
});
|
|
634
|
+
expect(split.tables).toHaveLength(2);
|
|
635
|
+
// Both tables match the event by ref, but only one may carry it.
|
|
636
|
+
const carriers = split.tables.filter((table) => table.syncEvent);
|
|
637
|
+
expect(carriers).toHaveLength(1);
|
|
638
|
+
|
|
639
|
+
// getSourceTableStatus preserves the carrier designation rather than recomputing it
|
|
640
|
+
// from the ref, so refreshing a non-carrier table does not make it fire events.
|
|
641
|
+
const nonCarrier = split.tables.find((table) => !table.syncEvent)!;
|
|
642
|
+
const refreshed = await writer.getSourceTableStatus(nonCarrier);
|
|
643
|
+
expect(refreshed!.syncEvent).toBe(false);
|
|
644
|
+
}
|
|
645
|
+
);
|
|
646
|
+
|
|
160
647
|
test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
|
|
161
648
|
await using factory = await storageConfig.factory();
|
|
162
649
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -173,7 +660,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
173
660
|
)
|
|
174
661
|
);
|
|
175
662
|
const bucketStorage = factory.getInstance(syncRules);
|
|
176
|
-
const
|
|
663
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
664
|
+
const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
|
|
177
665
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
178
666
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
179
667
|
|
|
@@ -194,7 +682,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
194
682
|
const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
|
|
195
683
|
const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
|
|
196
684
|
const buckets = await querier.queryDynamicBucketDescriptions({
|
|
197
|
-
async getParameterSets(lookups) {
|
|
685
|
+
async getParameterSets(lookups: ScopedParameterLookup[]) {
|
|
198
686
|
expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
|
|
199
687
|
expect(lookups[0].indexId).toEqual('1');
|
|
200
688
|
|
|
@@ -203,32 +691,663 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
203
691
|
return parameter_sets;
|
|
204
692
|
}
|
|
205
693
|
});
|
|
206
|
-
expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(
|
|
694
|
+
expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]);
|
|
207
695
|
|
|
208
696
|
const mongoFactory = factory as MongoBucketStorage;
|
|
209
697
|
const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
|
|
210
|
-
const currentDataCollections = await db.
|
|
698
|
+
const currentDataCollections = await db.listSourceRecordCollections(syncRules.replicationStreamId);
|
|
211
699
|
const currentData = await currentDataCollections[0]?.findOne({});
|
|
212
700
|
const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
|
|
213
701
|
expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
|
|
214
702
|
|
|
215
703
|
const bucketCollections = await mongoFactory.db.db
|
|
216
|
-
.listCollections({ name: new RegExp(`^bucket_data_${syncRules.
|
|
704
|
+
.listCollections({ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
|
|
217
705
|
.toArray();
|
|
218
706
|
expect(
|
|
219
|
-
bucketCollections.some(
|
|
707
|
+
bucketCollections.some(
|
|
708
|
+
(collection) => collection.name === `bucket_data_${syncRules.replicationStreamId}_${firstBucket?.def}`
|
|
709
|
+
)
|
|
220
710
|
).toBe(true);
|
|
221
711
|
|
|
222
|
-
const syncRule = await mongoFactory.db.sync_rules.findOne({
|
|
223
|
-
|
|
712
|
+
const syncRule = (await mongoFactory.db.sync_rules.findOne({
|
|
713
|
+
_id: syncRules.replicationStreamId
|
|
714
|
+
})) as ReplicationStreamDocumentV3;
|
|
715
|
+
const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
|
|
716
|
+
const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
|
|
224
717
|
expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
|
|
225
718
|
|
|
226
719
|
const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
|
|
227
720
|
expect(parameterIndexId).toBeDefined();
|
|
228
|
-
const parameterEntry = await db.
|
|
721
|
+
const parameterEntry = await db.parameterIndex(syncRules.replicationStreamId, parameterIndexId!).findOne({});
|
|
229
722
|
expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
|
|
230
723
|
});
|
|
231
724
|
|
|
725
|
+
test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => {
|
|
726
|
+
await using factory = await storageConfig.factory();
|
|
727
|
+
|
|
728
|
+
const first = await factory.updateSyncRules(
|
|
729
|
+
updateSyncRulesFromYaml(
|
|
730
|
+
`
|
|
731
|
+
config:
|
|
732
|
+
edition: 2
|
|
733
|
+
|
|
734
|
+
streams:
|
|
735
|
+
by_owner:
|
|
736
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
737
|
+
`,
|
|
738
|
+
{ storageVersion }
|
|
739
|
+
)
|
|
740
|
+
);
|
|
741
|
+
const second = await factory.updateSyncRules(
|
|
742
|
+
updateSyncRulesFromYaml(
|
|
743
|
+
`
|
|
744
|
+
config:
|
|
745
|
+
edition: 2
|
|
746
|
+
|
|
747
|
+
streams:
|
|
748
|
+
by_project:
|
|
749
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
750
|
+
`,
|
|
751
|
+
{ storageVersion }
|
|
752
|
+
)
|
|
753
|
+
);
|
|
754
|
+
|
|
755
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
756
|
+
expect(
|
|
757
|
+
(await factory.getStoppedReplicationStreams()).find(
|
|
758
|
+
(stream) => stream.replicationStreamId == first.replicationStreamId
|
|
759
|
+
)?.state
|
|
760
|
+
).toBe(storage.SyncRuleState.STOP);
|
|
761
|
+
|
|
762
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
763
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
764
|
+
expect(replicatingStreams[0].replicationStreamId).toEqual(second.replicationStreamId);
|
|
765
|
+
|
|
766
|
+
const configs = second.syncConfigContent;
|
|
767
|
+
expect(configs).toHaveLength(1);
|
|
768
|
+
const statuses = await Promise.all(configs.map((config) => config.getSyncConfigStatus()));
|
|
769
|
+
expect(statuses.map((status) => status?.state)).toEqual([storage.SyncRuleState.PROCESSING]);
|
|
770
|
+
expect(statuses.map((status) => status?.id)).toEqual(configs.map((config) => config.syncConfigId));
|
|
771
|
+
const parsed = replicatingStreams[0].parsed(test_utils.PARSE_OPTIONS);
|
|
772
|
+
expect(parsed.syncConfigs).toHaveLength(1);
|
|
773
|
+
expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
|
|
774
|
+
|
|
775
|
+
const bucketStorage = factory.getInstance(replicatingStreams[0]);
|
|
776
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
777
|
+
const resolved = await writer.resolveTables({
|
|
778
|
+
connection_id: 1,
|
|
779
|
+
source: sourceDescriptor('todos', { objectId: 'todos-relation' }),
|
|
780
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834b')
|
|
781
|
+
});
|
|
782
|
+
expect(resolved.tables).toHaveLength(1);
|
|
783
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => {
|
|
787
|
+
await using factory = await storageConfig.factory();
|
|
788
|
+
|
|
789
|
+
const first = await factory.updateSyncRules(
|
|
790
|
+
updateSyncRulesFromYaml(
|
|
791
|
+
`
|
|
792
|
+
config:
|
|
793
|
+
edition: 3
|
|
794
|
+
|
|
795
|
+
streams:
|
|
796
|
+
by_owner:
|
|
797
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
798
|
+
`,
|
|
799
|
+
{ storageVersion }
|
|
800
|
+
)
|
|
801
|
+
);
|
|
802
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
803
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
804
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
805
|
+
await firstWriter.commit('1/1');
|
|
806
|
+
|
|
807
|
+
const second = await factory.updateSyncRules(
|
|
808
|
+
updateSyncRulesFromYaml(
|
|
809
|
+
`
|
|
810
|
+
config:
|
|
811
|
+
edition: 3
|
|
812
|
+
|
|
813
|
+
streams:
|
|
814
|
+
by_project:
|
|
815
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
816
|
+
`,
|
|
817
|
+
{ storageVersion }
|
|
818
|
+
)
|
|
819
|
+
);
|
|
820
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
821
|
+
|
|
822
|
+
const bucketStorage = factory.getInstance(second);
|
|
823
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
824
|
+
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
825
|
+
|
|
826
|
+
const resolved = await writer.resolveTables({
|
|
827
|
+
connection_id: 1,
|
|
828
|
+
source,
|
|
829
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834c')
|
|
830
|
+
});
|
|
831
|
+
expect(resolved.tables).toHaveLength(1);
|
|
832
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(2);
|
|
833
|
+
|
|
834
|
+
await writer.markAllSnapshotDone('2/1');
|
|
835
|
+
await writer.commit('2/1');
|
|
836
|
+
|
|
837
|
+
const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
838
|
+
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
839
|
+
const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
|
|
840
|
+
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
841
|
+
});
|
|
842
|
+
|
|
843
|
+
test.runIf(storageVersion >= 3)(
|
|
844
|
+
'editing a stream query on the same table does not drop the active source table',
|
|
845
|
+
async () => {
|
|
846
|
+
await using factory = await storageConfig.factory();
|
|
847
|
+
|
|
848
|
+
const first = await factory.updateSyncRules(
|
|
849
|
+
updateSyncRulesFromYaml(
|
|
850
|
+
`
|
|
851
|
+
config:
|
|
852
|
+
edition: 3
|
|
853
|
+
|
|
854
|
+
streams:
|
|
855
|
+
todos:
|
|
856
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
857
|
+
`,
|
|
858
|
+
{ storageVersion }
|
|
859
|
+
)
|
|
860
|
+
);
|
|
861
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
862
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
863
|
+
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
864
|
+
const firstResolved = await firstWriter.resolveTables({
|
|
865
|
+
connection_id: 1,
|
|
866
|
+
source,
|
|
867
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
|
|
868
|
+
});
|
|
869
|
+
expect(firstResolved.dropTables).toHaveLength(0);
|
|
870
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
871
|
+
await firstWriter.commit('1/1');
|
|
872
|
+
|
|
873
|
+
const second = await factory.updateSyncRules(
|
|
874
|
+
updateSyncRulesFromYaml(
|
|
875
|
+
`
|
|
876
|
+
config:
|
|
877
|
+
edition: 3
|
|
878
|
+
|
|
879
|
+
streams:
|
|
880
|
+
todos:
|
|
881
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
882
|
+
`,
|
|
883
|
+
{ storageVersion }
|
|
884
|
+
)
|
|
885
|
+
);
|
|
886
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
887
|
+
|
|
888
|
+
const deploying = await factory.getDeployingSyncConfig();
|
|
889
|
+
expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
|
|
890
|
+
const bucketStorage = deploying!.storage;
|
|
891
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
892
|
+
const resolved = await writer.resolveTables({
|
|
893
|
+
connection_id: 1,
|
|
894
|
+
source,
|
|
895
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834e')
|
|
896
|
+
});
|
|
897
|
+
|
|
898
|
+
expect(resolved.dropTables).toHaveLength(0);
|
|
899
|
+
expect(resolved.tables.map((table) => table.id.toString())).toEqual(['6544e3899293153fa7b3834e']);
|
|
900
|
+
expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
|
|
901
|
+
|
|
902
|
+
const activeStatus = await firstWriter.getSourceTableStatus(firstResolved.tables[0]);
|
|
903
|
+
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
904
|
+
}
|
|
905
|
+
);
|
|
906
|
+
|
|
907
|
+
test.runIf(storageVersion >= 3)('reserves historical mapping ids without reusing stopped configs', async () => {
|
|
908
|
+
await using factory = await storageConfig.factory();
|
|
909
|
+
|
|
910
|
+
const ownerRules = `
|
|
911
|
+
config:
|
|
912
|
+
edition: 3
|
|
913
|
+
|
|
914
|
+
streams:
|
|
915
|
+
by_owner:
|
|
916
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
917
|
+
`;
|
|
918
|
+
const projectRules = `
|
|
919
|
+
config:
|
|
920
|
+
edition: 3
|
|
921
|
+
|
|
922
|
+
streams:
|
|
923
|
+
by_project:
|
|
924
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
925
|
+
`;
|
|
926
|
+
const statusRules = `
|
|
927
|
+
config:
|
|
928
|
+
edition: 3
|
|
929
|
+
|
|
930
|
+
streams:
|
|
931
|
+
by_status:
|
|
932
|
+
query: SELECT * FROM todos WHERE status = subscription.parameter('status')
|
|
933
|
+
`;
|
|
934
|
+
|
|
935
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
|
|
936
|
+
const firstStorage = factory.getInstance(first);
|
|
937
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
938
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
939
|
+
await firstWriter.commit('1/1');
|
|
940
|
+
|
|
941
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(projectRules, { storageVersion }));
|
|
942
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
943
|
+
|
|
944
|
+
const third = await factory.updateSyncRules(updateSyncRulesFromYaml(statusRules, { storageVersion }));
|
|
945
|
+
expect(third.replicationStreamId).toBe(first.replicationStreamId);
|
|
946
|
+
|
|
947
|
+
const configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
948
|
+
const statuses = await Promise.all(
|
|
949
|
+
configs.map(async (config) => [config, await config.getSyncConfigStatus()] as const)
|
|
950
|
+
);
|
|
951
|
+
// ownerRules
|
|
952
|
+
const activeConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.ACTIVE)?.[0];
|
|
953
|
+
// projectRules
|
|
954
|
+
const stoppedConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.STOP)?.[0];
|
|
955
|
+
// statusRules
|
|
956
|
+
const processingConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.PROCESSING)?.[0];
|
|
957
|
+
expect(stoppedConfig).toBeDefined();
|
|
958
|
+
expect(activeConfig).toBeDefined();
|
|
959
|
+
expect(processingConfig).toBeDefined();
|
|
960
|
+
|
|
961
|
+
const activePrefix = bucketRequest(activeConfig!, 'by_owner|0["owner"]').bucket;
|
|
962
|
+
const stoppedPrefix = bucketRequest(stoppedConfig!, 'by_project|0["project"]').bucket;
|
|
963
|
+
const processingPrefix = bucketRequest(processingConfig!, 'by_status|0["status"]').bucket;
|
|
964
|
+
|
|
965
|
+
expect(processingPrefix).not.toBe(stoppedPrefix);
|
|
966
|
+
expect(processingPrefix).not.toBe(activePrefix);
|
|
967
|
+
expect(bucketDefinitionId(processingPrefix)).toBeGreaterThan(
|
|
968
|
+
Math.max(bucketDefinitionId(stoppedPrefix), bucketDefinitionId(activePrefix))
|
|
969
|
+
);
|
|
970
|
+
});
|
|
971
|
+
|
|
972
|
+
test.runIf(storageVersion >= 3)('table snapshot status only affects sync configs using that table', async () => {
|
|
973
|
+
await using factory = await storageConfig.factory();
|
|
974
|
+
|
|
975
|
+
const first = await factory.updateSyncRules(
|
|
976
|
+
updateSyncRulesFromYaml(
|
|
977
|
+
`
|
|
978
|
+
config:
|
|
979
|
+
edition: 3
|
|
980
|
+
|
|
981
|
+
streams:
|
|
982
|
+
by_owner:
|
|
983
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
984
|
+
`,
|
|
985
|
+
{ storageVersion }
|
|
986
|
+
)
|
|
987
|
+
);
|
|
988
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
989
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
990
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
991
|
+
await firstWriter.commit('1/1');
|
|
992
|
+
|
|
993
|
+
const second = await factory.updateSyncRules(
|
|
994
|
+
updateSyncRulesFromYaml(
|
|
995
|
+
`
|
|
996
|
+
config:
|
|
997
|
+
edition: 3
|
|
998
|
+
|
|
999
|
+
streams:
|
|
1000
|
+
by_owner:
|
|
1001
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1002
|
+
by_project:
|
|
1003
|
+
query: SELECT * FROM projects WHERE id = subscription.parameter('project_id')
|
|
1004
|
+
`,
|
|
1005
|
+
{ storageVersion }
|
|
1006
|
+
)
|
|
1007
|
+
);
|
|
1008
|
+
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1009
|
+
|
|
1010
|
+
const bucketStorage = factory.getInstance(second);
|
|
1011
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1012
|
+
const resolved = await writer.resolveTables({
|
|
1013
|
+
connection_id: 1,
|
|
1014
|
+
source: sourceDescriptor('projects', { objectId: 'projects-relation' }),
|
|
1015
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
|
|
1016
|
+
});
|
|
1017
|
+
expect(resolved.tables).toHaveLength(1);
|
|
1018
|
+
|
|
1019
|
+
await writer.markAllSnapshotDone('2/1');
|
|
1020
|
+
await writer.markTableSnapshotRequired(resolved.tables[0]);
|
|
1021
|
+
|
|
1022
|
+
const configsAfterRequired = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1023
|
+
const statusesAfterRequired = await Promise.all(configsAfterRequired.map((config) => config.getSyncConfigStatus()));
|
|
1024
|
+
const ownerConfigAfterRequired = statusesAfterRequired.find(
|
|
1025
|
+
(status) => status?.state == storage.SyncRuleState.ACTIVE
|
|
1026
|
+
);
|
|
1027
|
+
const projectConfigAfterRequired = statusesAfterRequired.find(
|
|
1028
|
+
(status) => status?.state == storage.SyncRuleState.PROCESSING
|
|
1029
|
+
);
|
|
1030
|
+
expect(ownerConfigAfterRequired?.snapshot_done).toBe(true);
|
|
1031
|
+
expect(projectConfigAfterRequired?.snapshot_done).toBe(false);
|
|
1032
|
+
|
|
1033
|
+
await writer.markTableSnapshotDone(resolved.tables, '3/1');
|
|
1034
|
+
|
|
1035
|
+
const configsAfterDone = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1036
|
+
const statusesAfterDone = await Promise.all(configsAfterDone.map((config) => config.getSyncConfigStatus()));
|
|
1037
|
+
const ownerConfigAfterDone = statusesAfterDone.find((status) => status?.id == ownerConfigAfterRequired!.id);
|
|
1038
|
+
const projectConfigAfterDone = statusesAfterDone.find((status) => status?.id == projectConfigAfterRequired!.id);
|
|
1039
|
+
expect(ownerConfigAfterDone?.snapshot_done).toBe(true);
|
|
1040
|
+
expect(projectConfigAfterDone?.snapshot_done).toBe(false);
|
|
1041
|
+
});
|
|
1042
|
+
|
|
1043
|
+
test.runIf(storageVersion >= 3)(
|
|
1044
|
+
'keeps compatible active and deploying sync configs in one replication stream',
|
|
1045
|
+
async () => {
|
|
1046
|
+
await using factory = await storageConfig.factory();
|
|
1047
|
+
|
|
1048
|
+
const first = await factory.updateSyncRules(
|
|
1049
|
+
updateSyncRulesFromYaml(
|
|
1050
|
+
`
|
|
1051
|
+
config:
|
|
1052
|
+
edition: 3
|
|
1053
|
+
|
|
1054
|
+
streams:
|
|
1055
|
+
by_owner:
|
|
1056
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1057
|
+
`,
|
|
1058
|
+
{ storageVersion }
|
|
1059
|
+
)
|
|
1060
|
+
);
|
|
1061
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1062
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1063
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1064
|
+
await firstWriter.commit('1/1');
|
|
1065
|
+
const initialActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1066
|
+
|
|
1067
|
+
let configs = first.syncConfigContent;
|
|
1068
|
+
const firstConfigId = configs[0].syncConfigId;
|
|
1069
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
|
|
1070
|
+
|
|
1071
|
+
const second = await factory.updateSyncRules(
|
|
1072
|
+
updateSyncRulesFromYaml(
|
|
1073
|
+
`
|
|
1074
|
+
config:
|
|
1075
|
+
edition: 3
|
|
1076
|
+
|
|
1077
|
+
streams:
|
|
1078
|
+
by_project:
|
|
1079
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1080
|
+
`,
|
|
1081
|
+
{ storageVersion }
|
|
1082
|
+
)
|
|
1083
|
+
);
|
|
1084
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1085
|
+
|
|
1086
|
+
configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1087
|
+
const deploying = await factory.getDeployingSyncConfig();
|
|
1088
|
+
expect(configs).toHaveLength(2);
|
|
1089
|
+
expect(deploying).not.toBeNull();
|
|
1090
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
|
|
1091
|
+
|
|
1092
|
+
const stream = (await factory.getActiveSyncConfig())?.replicationStream;
|
|
1093
|
+
expect(stream?.state).toBe(storage.SyncRuleState.ACTIVE);
|
|
1094
|
+
expect(
|
|
1095
|
+
(await Promise.all(configs.map((config) => config.getSyncConfigStatus()))).map((status) => status?.state).sort()
|
|
1096
|
+
).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING]);
|
|
1097
|
+
|
|
1098
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1099
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1100
|
+
for (const config of configs) {
|
|
1101
|
+
expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
|
|
1105
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1106
|
+
await secondWriter.markAllSnapshotDone('2/1');
|
|
1107
|
+
await secondWriter.commit('2/1');
|
|
1108
|
+
|
|
1109
|
+
const updatedStream = (await factory.getActiveSyncConfig())?.replicationStream;
|
|
1110
|
+
expect(updatedStream?.state).toBe(storage.SyncRuleState.ACTIVE);
|
|
1111
|
+
const updatedConfigs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
|
|
1112
|
+
expect(
|
|
1113
|
+
(await Promise.all(updatedConfigs.map((config) => config.getSyncConfigStatus())))
|
|
1114
|
+
.map((status) => status?.state)
|
|
1115
|
+
.sort()
|
|
1116
|
+
).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.STOP]);
|
|
1117
|
+
expect(await factory.getDeployingSyncConfig()).toBeNull();
|
|
1118
|
+
expect((await factory.getActiveSyncConfig())?.content.syncConfigId).not.toBe(firstConfigId);
|
|
1119
|
+
const updatedActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1120
|
+
expect(updatedActiveStorage.replicationStream.replicationJobId).not.toBe(
|
|
1121
|
+
initialActiveStorage.replicationStream.replicationJobId
|
|
1122
|
+
);
|
|
1123
|
+
expect(updatedActiveStorage.replicationStream.replicationJobId).toContain(
|
|
1124
|
+
(await factory.getActiveSyncConfig())!.content.syncConfigId
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
);
|
|
1128
|
+
|
|
1129
|
+
test.runIf(storageVersion >= 3)(
|
|
1130
|
+
'appended compatible config adopts the stream checkpoint instead of regressing to 0',
|
|
1131
|
+
async () => {
|
|
1132
|
+
await using factory = await storageConfig.factory();
|
|
1133
|
+
|
|
1134
|
+
// First config replicates some data, advancing the stream-level op head and its checkpoint.
|
|
1135
|
+
const first = await factory.updateSyncRules(
|
|
1136
|
+
updateSyncRulesFromYaml(
|
|
1137
|
+
`
|
|
1138
|
+
config:
|
|
1139
|
+
edition: 3
|
|
1140
|
+
|
|
1141
|
+
streams:
|
|
1142
|
+
by_owner:
|
|
1143
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1144
|
+
`,
|
|
1145
|
+
{ storageVersion }
|
|
1146
|
+
)
|
|
1147
|
+
);
|
|
1148
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1149
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1150
|
+
const sourceTable = await test_utils.resolveTestTable(
|
|
1151
|
+
firstWriter,
|
|
1152
|
+
'todos',
|
|
1153
|
+
['id'],
|
|
1154
|
+
INITIALIZED_MONGO_STORAGE_FACTORY
|
|
1155
|
+
);
|
|
1156
|
+
await firstWriter.save({
|
|
1157
|
+
sourceTable,
|
|
1158
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1159
|
+
after: {
|
|
1160
|
+
id: 'todo-1',
|
|
1161
|
+
owner_id: 'user-1',
|
|
1162
|
+
project_id: 'project-1'
|
|
1163
|
+
},
|
|
1164
|
+
afterReplicaId: test_utils.rid('todo-1')
|
|
1165
|
+
});
|
|
1166
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1167
|
+
await firstWriter.commit('1/1');
|
|
1168
|
+
|
|
1169
|
+
const firstCheckpoint = (await firstStorage.getCheckpoint()).checkpoint;
|
|
1170
|
+
expect(firstCheckpoint).toBeGreaterThan(0n);
|
|
1171
|
+
|
|
1172
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1173
|
+
const streamDocBefore = (await mongoFactory.db.sync_rules.findOne({
|
|
1174
|
+
_id: first.replicationStreamId
|
|
1175
|
+
})) as ReplicationStreamDocumentV3;
|
|
1176
|
+
// The stream-level head was advanced durably to (at least) the first config's checkpoint.
|
|
1177
|
+
expect(streamDocBefore.last_persisted_op).not.toBeNull();
|
|
1178
|
+
expect(BigInt(streamDocBefore.last_persisted_op!)).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1179
|
+
|
|
1180
|
+
// Append a compatible second config that replicates nothing new.
|
|
1181
|
+
const second = await factory.updateSyncRules(
|
|
1182
|
+
updateSyncRulesFromYaml(
|
|
1183
|
+
`
|
|
1184
|
+
config:
|
|
1185
|
+
edition: 3
|
|
1186
|
+
|
|
1187
|
+
streams:
|
|
1188
|
+
by_project:
|
|
1189
|
+
query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
|
|
1190
|
+
`,
|
|
1191
|
+
{ storageVersion }
|
|
1192
|
+
)
|
|
1193
|
+
);
|
|
1194
|
+
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1195
|
+
|
|
1196
|
+
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1197
|
+
expect(replicatingStreams).toHaveLength(1);
|
|
1198
|
+
const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
|
|
1199
|
+
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1200
|
+
// No new data replicated - just complete the snapshot and commit.
|
|
1201
|
+
await secondWriter.markAllSnapshotDone('2/1');
|
|
1202
|
+
await secondWriter.commit('2/1');
|
|
1203
|
+
|
|
1204
|
+
// The appended config's checkpoint must adopt the stream head, not regress to 0.
|
|
1205
|
+
const streamDocAfter = (await mongoFactory.db.sync_rules.findOne({
|
|
1206
|
+
_id: first.replicationStreamId
|
|
1207
|
+
})) as ReplicationStreamDocumentV3;
|
|
1208
|
+
const head = BigInt(streamDocAfter.last_persisted_op!);
|
|
1209
|
+
for (const config of streamDocAfter.sync_configs) {
|
|
1210
|
+
expect(config.last_checkpoint).not.toBeNull();
|
|
1211
|
+
expect(BigInt(config.last_checkpoint!)).toEqual(head);
|
|
1212
|
+
}
|
|
1213
|
+
// All configs share the same checkpoint, equal to the stream head.
|
|
1214
|
+
expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1215
|
+
|
|
1216
|
+
// After activation, the active config's checkpoint does not regress.
|
|
1217
|
+
const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
|
|
1218
|
+
const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
|
|
1219
|
+
expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1220
|
+
}
|
|
1221
|
+
);
|
|
1222
|
+
|
|
1223
|
+
test.runIf(storageVersion >= 3)('creates a new replication stream when compatibility options differ', async () => {
|
|
1224
|
+
await using factory = await storageConfig.factory();
|
|
1225
|
+
|
|
1226
|
+
const first = await factory.updateSyncRules(
|
|
1227
|
+
updateSyncRulesFromYaml(
|
|
1228
|
+
`
|
|
1229
|
+
config:
|
|
1230
|
+
edition: 3
|
|
1231
|
+
|
|
1232
|
+
streams:
|
|
1233
|
+
by_owner:
|
|
1234
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1235
|
+
`,
|
|
1236
|
+
{ storageVersion }
|
|
1237
|
+
)
|
|
1238
|
+
);
|
|
1239
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1240
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1241
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1242
|
+
await firstWriter.commit('1/1');
|
|
1243
|
+
|
|
1244
|
+
// Same streams, but a different compatibility edition - must not append to the active stream.
|
|
1245
|
+
const second = await factory.updateSyncRules(
|
|
1246
|
+
updateSyncRulesFromYaml(
|
|
1247
|
+
`
|
|
1248
|
+
config:
|
|
1249
|
+
edition: 2
|
|
1250
|
+
|
|
1251
|
+
streams:
|
|
1252
|
+
by_owner:
|
|
1253
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1254
|
+
`,
|
|
1255
|
+
{ storageVersion }
|
|
1256
|
+
)
|
|
1257
|
+
);
|
|
1258
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1259
|
+
|
|
1260
|
+
// The first stream stays active until the replacement has replicated.
|
|
1261
|
+
const active = await factory.getActiveSyncConfig();
|
|
1262
|
+
expect(active?.replicationStream.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1263
|
+
expect((await factory.getDeployingSyncConfig())?.replicationStream.replicationStreamId).toEqual(
|
|
1264
|
+
second.replicationStreamId
|
|
1265
|
+
);
|
|
1266
|
+
});
|
|
1267
|
+
|
|
1268
|
+
test.runIf(storageVersion >= 3)(
|
|
1269
|
+
'creates a new replication stream for legacy sync rules without a serialized plan',
|
|
1270
|
+
async () => {
|
|
1271
|
+
await using factory = await storageConfig.factory();
|
|
1272
|
+
|
|
1273
|
+
const first = await factory.updateSyncRules(
|
|
1274
|
+
updateSyncRulesFromYaml(
|
|
1275
|
+
`
|
|
1276
|
+
config:
|
|
1277
|
+
edition: 3
|
|
1278
|
+
|
|
1279
|
+
streams:
|
|
1280
|
+
by_owner:
|
|
1281
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1282
|
+
`,
|
|
1283
|
+
{ storageVersion }
|
|
1284
|
+
)
|
|
1285
|
+
);
|
|
1286
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1287
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1288
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1289
|
+
await firstWriter.commit('1/1');
|
|
1290
|
+
|
|
1291
|
+
// Legacy sync rules have no serialized plan - must not append to the active stream.
|
|
1292
|
+
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1293
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1294
|
+
}
|
|
1295
|
+
);
|
|
1296
|
+
|
|
1297
|
+
test.runIf(storageVersion >= 3)('does not append to an active legacy sync config', async () => {
|
|
1298
|
+
await using factory = await storageConfig.factory();
|
|
1299
|
+
|
|
1300
|
+
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1301
|
+
const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
|
|
1302
|
+
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1303
|
+
await firstWriter.markAllSnapshotDone('1/1');
|
|
1304
|
+
await firstWriter.commit('1/1');
|
|
1305
|
+
|
|
1306
|
+
// The active config has no serialized plan, so a new streams config must not be appended to it.
|
|
1307
|
+
const second = await factory.updateSyncRules(
|
|
1308
|
+
updateSyncRulesFromYaml(
|
|
1309
|
+
`
|
|
1310
|
+
config:
|
|
1311
|
+
edition: 3
|
|
1312
|
+
|
|
1313
|
+
streams:
|
|
1314
|
+
by_owner:
|
|
1315
|
+
query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
|
|
1316
|
+
`,
|
|
1317
|
+
{ storageVersion }
|
|
1318
|
+
)
|
|
1319
|
+
);
|
|
1320
|
+
expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
|
|
1324
|
+
await using factory = await storageConfig.factory();
|
|
1325
|
+
|
|
1326
|
+
const firstSyncRules = await factory.updateSyncRules(
|
|
1327
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion })
|
|
1328
|
+
);
|
|
1329
|
+
|
|
1330
|
+
await expect(
|
|
1331
|
+
factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }))
|
|
1332
|
+
).resolves.toBeDefined();
|
|
1333
|
+
|
|
1334
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1335
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.replicationStreamId }))?.state).toBe(
|
|
1336
|
+
storage.SyncRuleState.STOP
|
|
1337
|
+
);
|
|
1338
|
+
});
|
|
1339
|
+
|
|
1340
|
+
test('can lock newly-created sync rules', async () => {
|
|
1341
|
+
await using factory = await storageConfig.factory();
|
|
1342
|
+
|
|
1343
|
+
const syncRules = await factory.updateSyncRules(
|
|
1344
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
|
|
1345
|
+
);
|
|
1346
|
+
|
|
1347
|
+
expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.replicationStreamId);
|
|
1348
|
+
await syncRules.current_lock?.release();
|
|
1349
|
+
});
|
|
1350
|
+
|
|
232
1351
|
test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
|
|
233
1352
|
await using factory = await storageConfig.factory();
|
|
234
1353
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -260,10 +1379,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
260
1379
|
await writer.commit('1/1');
|
|
261
1380
|
|
|
262
1381
|
const mongoFactory = factory as MongoBucketStorage;
|
|
263
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1382
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
264
1383
|
|
|
265
1384
|
const sourceRecordCollections = await mongoFactory.db.db
|
|
266
|
-
.listCollections({ name: new RegExp(`^source_records_${syncRules.
|
|
1385
|
+
.listCollections({ name: new RegExp(`^source_records_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
|
|
267
1386
|
.toArray();
|
|
268
1387
|
expect(sourceRecordCollections).toEqual([]);
|
|
269
1388
|
});
|
|
@@ -299,11 +1418,11 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
299
1418
|
await writer.commit('1/1');
|
|
300
1419
|
|
|
301
1420
|
const mongoFactory = factory as MongoBucketStorage;
|
|
302
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1421
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
303
1422
|
|
|
304
1423
|
await bucketStorage.clear();
|
|
305
1424
|
|
|
306
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1425
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
|
|
307
1426
|
});
|
|
308
1427
|
|
|
309
1428
|
test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
|
|
@@ -338,7 +1457,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
338
1457
|
await writer.commit('1/1');
|
|
339
1458
|
|
|
340
1459
|
const mongoFactory = factory as MongoBucketStorage;
|
|
341
|
-
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.
|
|
1460
|
+
expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
|
|
342
1461
|
|
|
343
1462
|
const metricsAfter = await factory.getStorageMetrics();
|
|
344
1463
|
expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
|
|
@@ -396,7 +1515,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
396
1515
|
}
|
|
397
1516
|
);
|
|
398
1517
|
|
|
399
|
-
test.runIf(storageVersion
|
|
1518
|
+
test.runIf(storageVersion == 3)('cleans pending deletes only for tracked v3 source tables', async () => {
|
|
400
1519
|
await using factory = await storageConfig.factory();
|
|
401
1520
|
const syncRules = await factory.updateSyncRules(
|
|
402
1521
|
updateSyncRulesFromYaml(
|
|
@@ -411,21 +1530,20 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
411
1530
|
);
|
|
412
1531
|
|
|
413
1532
|
const mongoFactory = factory as MongoBucketStorage;
|
|
414
|
-
const bucketStorage = mongoFactory.getInstance(syncRules)
|
|
415
|
-
const db = bucketStorage.db;
|
|
416
|
-
await db.initializeStreamStorage(syncRules.
|
|
1533
|
+
const bucketStorage = mongoFactory.getInstance(syncRules);
|
|
1534
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1535
|
+
await db.initializeStreamStorage(syncRules.replicationStreamId);
|
|
417
1536
|
|
|
418
1537
|
const sourceTableA = new bson.ObjectId();
|
|
419
1538
|
const sourceTableB = new bson.ObjectId();
|
|
420
|
-
await db.
|
|
1539
|
+
await db.sourceTables(syncRules.replicationStreamId).insertMany([
|
|
421
1540
|
{
|
|
422
1541
|
_id: sourceTableA,
|
|
423
1542
|
connection_id: 1,
|
|
424
1543
|
relation_id: 'a',
|
|
425
1544
|
schema_name: 'public',
|
|
426
1545
|
table_name: 'table_a',
|
|
427
|
-
replica_id_columns:
|
|
428
|
-
replica_id_columns2: [],
|
|
1546
|
+
replica_id_columns: [],
|
|
429
1547
|
snapshot_done: true,
|
|
430
1548
|
snapshot_status: undefined,
|
|
431
1549
|
bucket_data_source_ids: [],
|
|
@@ -438,8 +1556,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
438
1556
|
relation_id: 'b',
|
|
439
1557
|
schema_name: 'public',
|
|
440
1558
|
table_name: 'table_b',
|
|
441
|
-
replica_id_columns:
|
|
442
|
-
replica_id_columns2: [],
|
|
1559
|
+
replica_id_columns: [],
|
|
443
1560
|
snapshot_done: true,
|
|
444
1561
|
snapshot_status: undefined,
|
|
445
1562
|
bucket_data_source_ids: [],
|
|
@@ -448,42 +1565,312 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
448
1565
|
}
|
|
449
1566
|
]);
|
|
450
1567
|
|
|
451
|
-
await db.
|
|
1568
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).insertMany([
|
|
452
1569
|
{ _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
|
|
453
1570
|
{ _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
|
|
454
1571
|
{ _id: 'active', data: null, buckets: [], lookups: [] }
|
|
455
1572
|
]);
|
|
456
1573
|
await db
|
|
457
|
-
.
|
|
1574
|
+
.sourceRecords(syncRules.replicationStreamId, sourceTableB)
|
|
458
1575
|
.insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
|
|
459
1576
|
|
|
460
|
-
const store = new SourceRecordStoreV3(
|
|
1577
|
+
const store = new SourceRecordStoreV3(
|
|
1578
|
+
db,
|
|
1579
|
+
syncRules.replicationStreamId,
|
|
1580
|
+
bucketStorage.replicationStream.syncConfigContent[0].mapping
|
|
1581
|
+
);
|
|
461
1582
|
const logger = { info() {} } as any;
|
|
462
1583
|
|
|
463
1584
|
await store.postCommitCleanup(6n, logger);
|
|
464
1585
|
|
|
465
|
-
expect(
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
expect(
|
|
469
|
-
|
|
1586
|
+
expect(
|
|
1587
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 5n })
|
|
1588
|
+
).toBe(0);
|
|
1589
|
+
expect(
|
|
1590
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 9n })
|
|
1591
|
+
).toBe(1);
|
|
1592
|
+
expect(
|
|
1593
|
+
await db.sourceRecords(syncRules.replicationStreamId, sourceTableB).countDocuments({ pending_delete: 12n })
|
|
1594
|
+
).toBe(1);
|
|
1595
|
+
expect(
|
|
1596
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
|
|
1597
|
+
).toBe(9n);
|
|
1598
|
+
expect(
|
|
1599
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
|
|
1600
|
+
).toBe(12n);
|
|
470
1601
|
|
|
471
1602
|
await store.postCommitCleanup(10n, logger);
|
|
472
1603
|
|
|
473
1604
|
expect(
|
|
474
|
-
await db
|
|
1605
|
+
await db
|
|
1606
|
+
.sourceRecords(syncRules.replicationStreamId, sourceTableA)
|
|
1607
|
+
.countDocuments({ pending_delete: { $exists: true } })
|
|
475
1608
|
).toBe(0);
|
|
476
1609
|
expect(
|
|
477
|
-
(await db.
|
|
1610
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
|
|
478
1611
|
).toBeUndefined();
|
|
479
|
-
expect(
|
|
1612
|
+
expect(
|
|
1613
|
+
(await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
|
|
1614
|
+
).toBe(12n);
|
|
480
1615
|
});
|
|
481
1616
|
}
|
|
482
1617
|
|
|
483
1618
|
describe('sync - mongodb', () => {
|
|
1619
|
+
test('v3 activation stops legacy active sync rules', async () => {
|
|
1620
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1621
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
1622
|
+
|
|
1623
|
+
const legacySyncRules = await factory.updateSyncRules(
|
|
1624
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.LEGACY_STORAGE_VERSION })
|
|
1625
|
+
);
|
|
1626
|
+
const legacyStorage = factory.getInstance(legacySyncRules);
|
|
1627
|
+
await using legacyWriter = await legacyStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1628
|
+
await legacyWriter.markAllSnapshotDone('1/1');
|
|
1629
|
+
await legacyWriter.commit('1/1');
|
|
1630
|
+
|
|
1631
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
|
|
1632
|
+
storage.SyncRuleState.ACTIVE
|
|
1633
|
+
);
|
|
1634
|
+
|
|
1635
|
+
const v3SyncRules = await factory.updateSyncRules(
|
|
1636
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.STORAGE_VERSION_3 })
|
|
1637
|
+
);
|
|
1638
|
+
const v3Storage = factory.getInstance(v3SyncRules);
|
|
1639
|
+
await using v3Writer = await v3Storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1640
|
+
await v3Writer.markAllSnapshotDone('2/1');
|
|
1641
|
+
await v3Writer.commit('2/1');
|
|
1642
|
+
|
|
1643
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
|
|
1644
|
+
storage.SyncRuleState.STOP
|
|
1645
|
+
);
|
|
1646
|
+
});
|
|
1647
|
+
|
|
484
1648
|
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
485
1649
|
describe(`storage v${storageVersion}`, () => {
|
|
486
1650
|
registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
|
|
1651
|
+
|
|
1652
|
+
describe.runIf(storageVersion == 3)('V3 read filtering boundaries', () => {
|
|
1653
|
+
async function setupFilteringTest() {
|
|
1654
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1655
|
+
const syncRules = await factory.updateSyncRules(
|
|
1656
|
+
updateSyncRulesFromYaml(
|
|
1657
|
+
`
|
|
1658
|
+
bucket_definitions:
|
|
1659
|
+
global:
|
|
1660
|
+
data:
|
|
1661
|
+
- SELECT id, description FROM test
|
|
1662
|
+
`,
|
|
1663
|
+
{ storageVersion }
|
|
1664
|
+
)
|
|
1665
|
+
);
|
|
1666
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
1667
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1668
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
1669
|
+
|
|
1670
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
1671
|
+
const definitionId = mapping.bucketSourceId(request.source);
|
|
1672
|
+
const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
|
|
1673
|
+
|
|
1674
|
+
const bucketName = request.bucket;
|
|
1675
|
+
const sourceTable = new bson.ObjectId();
|
|
1676
|
+
const bucketKey: BucketKey = {
|
|
1677
|
+
replicationStreamId: syncRules.replicationStreamId,
|
|
1678
|
+
definitionId,
|
|
1679
|
+
bucket: bucketName
|
|
1680
|
+
};
|
|
1681
|
+
|
|
1682
|
+
function makeOps(opIds: bigint[]): BucketDataDoc[] {
|
|
1683
|
+
return opIds.map((opId) => ({
|
|
1684
|
+
bucketKey,
|
|
1685
|
+
o: opId,
|
|
1686
|
+
op: 'PUT' as const,
|
|
1687
|
+
source_table: sourceTable,
|
|
1688
|
+
source_key: test_utils.rid(`row-${opId}`),
|
|
1689
|
+
table: 'items',
|
|
1690
|
+
row_id: `row-${opId}`,
|
|
1691
|
+
checksum: BigInt(opId) * 10n,
|
|
1692
|
+
data: `{"id":"row-${opId}"}`
|
|
1693
|
+
}));
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
const docA = serializeBucketData(bucketName, makeOps([10n, 20n, 30n]));
|
|
1697
|
+
const docB = serializeBucketData(bucketName, makeOps([40n, 50n, 60n]));
|
|
1698
|
+
const docC = serializeBucketData(bucketName, makeOps([70n, 80n, 90n]));
|
|
1699
|
+
|
|
1700
|
+
await collection.insertMany([docA, docB, docC]);
|
|
1701
|
+
|
|
1702
|
+
return { factory, syncRules, bucketStorage, bucketName };
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
|
|
1706
|
+
const { syncRules, bucketStorage } = await setupFilteringTest();
|
|
1707
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
|
|
1708
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(BigInt(checkpoint), [request]));
|
|
1709
|
+
const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
|
|
1710
|
+
return ops;
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
test('case 1: start=5, checkpoint=95 → all ops', async () => {
|
|
1714
|
+
const ops = await getFilteredOps(5, 95);
|
|
1715
|
+
expect(ops).toEqual([10n, 20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
|
|
1716
|
+
});
|
|
1717
|
+
|
|
1718
|
+
test('case 2: start=10, checkpoint=90 → ops in (10,90]', async () => {
|
|
1719
|
+
const ops = await getFilteredOps(10, 90);
|
|
1720
|
+
expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
|
|
1721
|
+
});
|
|
1722
|
+
|
|
1723
|
+
test('case 3: start=15, checkpoint=85 → partial doc boundaries', async () => {
|
|
1724
|
+
const ops = await getFilteredOps(15, 85);
|
|
1725
|
+
expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n]);
|
|
1726
|
+
});
|
|
1727
|
+
|
|
1728
|
+
test('case 4: start=25, checkpoint=55 → spans two docs', async () => {
|
|
1729
|
+
const ops = await getFilteredOps(25, 55);
|
|
1730
|
+
expect(ops).toEqual([30n, 40n, 50n]);
|
|
1731
|
+
});
|
|
1732
|
+
|
|
1733
|
+
test('case 5: start=35, checkpoint=45 → single op within doc', async () => {
|
|
1734
|
+
const ops = await getFilteredOps(35, 45);
|
|
1735
|
+
expect(ops).toEqual([40n]);
|
|
1736
|
+
});
|
|
1737
|
+
|
|
1738
|
+
test('case 6: start=35, checkpoint=65 → full doc B', async () => {
|
|
1739
|
+
const ops = await getFilteredOps(35, 65);
|
|
1740
|
+
expect(ops).toEqual([40n, 50n, 60n]);
|
|
1741
|
+
});
|
|
1742
|
+
|
|
1743
|
+
test('case 7: start=25, checkpoint=35 → single op from doc A', async () => {
|
|
1744
|
+
const ops = await getFilteredOps(25, 35);
|
|
1745
|
+
expect(ops).toEqual([30n]);
|
|
1746
|
+
});
|
|
1747
|
+
|
|
1748
|
+
test('case 8: start=30, checkpoint=40 → op at checkpoint from next doc', async () => {
|
|
1749
|
+
const ops = await getFilteredOps(30, 40);
|
|
1750
|
+
expect(ops).toEqual([40n]);
|
|
1751
|
+
});
|
|
1752
|
+
|
|
1753
|
+
test('case 9: start=100, checkpoint=200 → beyond all docs', async () => {
|
|
1754
|
+
const ops = await getFilteredOps(100, 200);
|
|
1755
|
+
expect(ops).toEqual([]);
|
|
1756
|
+
});
|
|
1757
|
+
|
|
1758
|
+
test('case 10: start=0, checkpoint=5 → before all docs', async () => {
|
|
1759
|
+
const ops = await getFilteredOps(0, 5);
|
|
1760
|
+
expect(ops).toEqual([]);
|
|
1761
|
+
});
|
|
1762
|
+
|
|
1763
|
+
test('case 11: start=50, checkpoint=50 → zero-width range', async () => {
|
|
1764
|
+
const ops = await getFilteredOps(50, 50);
|
|
1765
|
+
expect(ops).toEqual([]);
|
|
1766
|
+
});
|
|
1767
|
+
|
|
1768
|
+
test('case 12: start=45, checkpoint=50 → op at checkpoint boundary', async () => {
|
|
1769
|
+
const ops = await getFilteredOps(45, 50);
|
|
1770
|
+
expect(ops).toEqual([50n]);
|
|
1771
|
+
});
|
|
1772
|
+
|
|
1773
|
+
test('case 13: start=50, checkpoint=55 → no ops strictly after start', async () => {
|
|
1774
|
+
const ops = await getFilteredOps(50, 55);
|
|
1775
|
+
expect(ops).toEqual([]);
|
|
1776
|
+
});
|
|
1777
|
+
|
|
1778
|
+
test('all-filtered first batch still returns data behind the batch boundary', async () => {
|
|
1779
|
+
// Documents straddling the requested (start, end] window are matched by the
|
|
1780
|
+
// query, but contribute no rows after filtering. If an entire server batch
|
|
1781
|
+
// (~101 documents) consists of such straddlers, the remaining documents in
|
|
1782
|
+
// the cursor must still be reachable. Storage reports the straddler buckets
|
|
1783
|
+
// as complete via empty chunks, and the caller re-requests the rest.
|
|
1784
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1785
|
+
const syncRules = await factory.updateSyncRules(
|
|
1786
|
+
updateSyncRulesFromYaml(
|
|
1787
|
+
`
|
|
1788
|
+
bucket_definitions:
|
|
1789
|
+
by_user:
|
|
1790
|
+
parameters: select request.user_id() as user_id
|
|
1791
|
+
data: [select * from test where owner_id = bucket.user_id]
|
|
1792
|
+
`,
|
|
1793
|
+
{ storageVersion }
|
|
1794
|
+
)
|
|
1795
|
+
);
|
|
1796
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3;
|
|
1797
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1798
|
+
|
|
1799
|
+
const start = 5n;
|
|
1800
|
+
const end = 50n;
|
|
1801
|
+
|
|
1802
|
+
// 150 buckets sorted before the data bucket, each with a single document
|
|
1803
|
+
// containing ops at 1 and 100: matched (_id.o=100 > start, min_op=1 <= end),
|
|
1804
|
+
// but no op in (5, 50].
|
|
1805
|
+
const straddlerNames = Array.from({ length: 150 }, (_, i) => `b${`${i}`.padStart(3, '0')}`);
|
|
1806
|
+
const requests = [...straddlerNames, 'zzz'].map((id) =>
|
|
1807
|
+
bucketRequest(syncRules.syncConfigContent[0], `by_user["${id}"]`, start)
|
|
1808
|
+
);
|
|
1809
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
1810
|
+
const definitionId = mapping.bucketSourceId(requests[0].source);
|
|
1811
|
+
const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
|
|
1812
|
+
const sourceTable = new bson.ObjectId();
|
|
1813
|
+
|
|
1814
|
+
function makeOps(bucket: string, opIds: bigint[]): BucketDataDoc[] {
|
|
1815
|
+
const bucketKey: BucketKey = {
|
|
1816
|
+
replicationStreamId: syncRules.replicationStreamId,
|
|
1817
|
+
definitionId,
|
|
1818
|
+
bucket
|
|
1819
|
+
};
|
|
1820
|
+
return opIds.map((opId) => ({
|
|
1821
|
+
bucketKey,
|
|
1822
|
+
o: opId,
|
|
1823
|
+
op: 'PUT' as const,
|
|
1824
|
+
source_table: sourceTable,
|
|
1825
|
+
source_key: test_utils.rid(`row-${opId}`),
|
|
1826
|
+
table: 'test',
|
|
1827
|
+
row_id: `row-${opId}`,
|
|
1828
|
+
checksum: BigInt(opId) * 10n,
|
|
1829
|
+
data: `{"id":"row-${opId}"}`
|
|
1830
|
+
}));
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
const straddlerDocs = requests
|
|
1834
|
+
.slice(0, -1)
|
|
1835
|
+
.map((request) => serializeBucketData(request.bucket, makeOps(request.bucket, [1n, 100n])));
|
|
1836
|
+
const dataBucket = requests[requests.length - 1].bucket;
|
|
1837
|
+
const dataDoc = serializeBucketData(dataBucket, makeOps(dataBucket, [10n]));
|
|
1838
|
+
await collection.insertMany([...straddlerDocs, dataDoc]);
|
|
1839
|
+
|
|
1840
|
+
// Emulate the caller loop in sync.ts / BucketChecksumState: advance bucket
|
|
1841
|
+
// positions from each chunk, drop completed buckets, and re-request while
|
|
1842
|
+
// any chunk reported has_more.
|
|
1843
|
+
const positions = new Map(requests.map((request) => [request.bucket, request.start]));
|
|
1844
|
+
const pending = new Set(positions.keys());
|
|
1845
|
+
const receivedOps: bigint[] = [];
|
|
1846
|
+
let rounds = 0;
|
|
1847
|
+
|
|
1848
|
+
while (rounds < 10) {
|
|
1849
|
+
rounds++;
|
|
1850
|
+
const roundRequests = requests
|
|
1851
|
+
.filter((request) => pending.has(request.bucket))
|
|
1852
|
+
.map((request) => ({ ...request, start: positions.get(request.bucket)! }));
|
|
1853
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(end, roundRequests));
|
|
1854
|
+
let anyHasMore = false;
|
|
1855
|
+
for (const { chunkData } of batch) {
|
|
1856
|
+
positions.set(chunkData.bucket, BigInt(chunkData.next_after));
|
|
1857
|
+
if (chunkData.has_more) {
|
|
1858
|
+
anyHasMore = true;
|
|
1859
|
+
} else {
|
|
1860
|
+
pending.delete(chunkData.bucket);
|
|
1861
|
+
}
|
|
1862
|
+
receivedOps.push(...chunkData.data.map((entry) => BigInt(entry.op_id)));
|
|
1863
|
+
}
|
|
1864
|
+
if (!anyHasMore) {
|
|
1865
|
+
break;
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
// The op behind the all-straddler first batch must be returned.
|
|
1870
|
+
expect(receivedOps).toEqual([10n]);
|
|
1871
|
+
expect(rounds).toBeLessThan(10);
|
|
1872
|
+
});
|
|
1873
|
+
});
|
|
487
1874
|
});
|
|
488
1875
|
}
|
|
489
1876
|
});
|