@powersync/service-module-mongodb-storage 0.22.0 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/dist/storage/MongoBucketStorage.d.ts +5 -1
- package/dist/storage/MongoBucketStorage.js +30 -6
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +23 -10
- package/dist/storage/implementation/MongoBucketBatch.js +132 -88
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +2 -0
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +8 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +5 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.js +17 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +38 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/db.js +1 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +5 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +136 -133
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +5 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +131 -148
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +4 -3
- package/dist/storage/implementation/v3/MongoCompactorV3.js +19 -10
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +6 -3
- package/dist/storage/implementation/v3/compact-utils.js +12 -5
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +1 -1
- package/dist/utils/test-utils.d.ts +2 -1
- package/dist/utils/test-utils.js +4 -3
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +7 -7
- package/src/storage/MongoBucketStorage.ts +35 -7
- package/src/storage/implementation/MongoBucketBatch.ts +201 -110
- package/src/storage/implementation/MongoCompactor.ts +2 -0
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +19 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +19 -0
- package/src/storage/implementation/MongoSyncRulesLock.ts +56 -3
- package/src/storage/implementation/db.ts +1 -0
- package/src/storage/implementation/models.ts +1 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +150 -152
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +1 -1
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +167 -194
- package/src/storage/implementation/v3/MongoCompactorV3.ts +25 -22
- package/src/storage/implementation/v3/compact-utils.ts +19 -8
- package/src/storage/implementation/v3/models.ts +1 -1
- package/src/utils/test-utils.ts +5 -3
- package/test/src/__snapshots__/storage.test.ts.snap +102 -2
- package/test/src/__snapshots__/storage_compacting.test.ts.snap +2 -2
- package/test/src/__snapshots__/storage_sync.test.ts.snap +12 -12
- package/test/src/cleanup-stopped-sync-configs.test.ts +19 -13
- package/test/src/compact-utils.test.ts +33 -8
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +2 -1
- package/test/src/parameter_compacting_v1.test.ts +2 -2
- package/test/src/parameter_compacting_v3.test.ts +2 -2
- package/test/src/parameter_compaction_fence.test.ts +3 -3
- package/test/src/source_table_config.test.ts +97 -0
- package/test/src/storage.test.ts +32 -33
- package/test/src/storage_compacting.test.ts +184 -21
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +4 -4
- package/test/src/storage_s3_compaction_lifecycle.test.ts +8 -8
- package/test/src/storage_s3_reading.test.ts +4 -4
- package/test/src/storage_s3_writing.test.ts +3 -3
- package/test/src/storage_sync.test.ts +54 -47
- package/test/src/storeCurrentData.test.ts +5 -5
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
storage,
|
|
7
7
|
updateSyncRulesFromYaml
|
|
8
8
|
} from '@powersync/service-core';
|
|
9
|
-
import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
|
|
9
|
+
import { bucketRequest, getTestStorage, register, test_utils } from '@powersync/service-core-tests';
|
|
10
10
|
import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
11
11
|
import * as bson from 'bson';
|
|
12
12
|
import { describe, expect, test } from 'vitest';
|
|
@@ -73,7 +73,8 @@ function objectIdGenerator(id: string) {
|
|
|
73
73
|
*/
|
|
74
74
|
function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
|
|
75
75
|
const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
|
|
76
|
-
|
|
76
|
+
// These fixtures are legacy Sync Rules, which always report a deprecation warning.
|
|
77
|
+
expect(parsed.errors.filter((error) => error.type !== 'warning')).toEqual([]);
|
|
77
78
|
return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
|
|
78
79
|
{
|
|
79
80
|
syncConfigId: new bson.ObjectId().toHexString(),
|
|
@@ -149,7 +150,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
149
150
|
test('updates source metadata on an existing resolved table', async () => {
|
|
150
151
|
await using factory = await storageConfig.factory();
|
|
151
152
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
152
|
-
const bucketStorage = factory
|
|
153
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
153
154
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
154
155
|
const source = sourceDescriptor('test');
|
|
155
156
|
|
|
@@ -191,7 +192,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
191
192
|
{ storageVersion }
|
|
192
193
|
)
|
|
193
194
|
);
|
|
194
|
-
const bucketStorage = factory
|
|
195
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
195
196
|
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
196
197
|
|
|
197
198
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -338,7 +339,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
338
339
|
{ storageVersion }
|
|
339
340
|
)
|
|
340
341
|
);
|
|
341
|
-
const bucketStorage = factory
|
|
342
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
342
343
|
|
|
343
344
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
344
345
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -363,7 +364,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
363
364
|
{ storageVersion }
|
|
364
365
|
)
|
|
365
366
|
);
|
|
366
|
-
const bucketStorage = factory
|
|
367
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
367
368
|
|
|
368
369
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
369
370
|
const before = await writer.resolveTables({
|
|
@@ -401,7 +402,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
401
402
|
{ storageVersion }
|
|
402
403
|
)
|
|
403
404
|
);
|
|
404
|
-
const bucketStorage = factory
|
|
405
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
405
406
|
|
|
406
407
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
407
408
|
const before = await writer.resolveTables({
|
|
@@ -437,7 +438,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
437
438
|
{ storageVersion }
|
|
438
439
|
)
|
|
439
440
|
);
|
|
440
|
-
const bucketStorage = factory
|
|
441
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
441
442
|
|
|
442
443
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
443
444
|
const before = await writer.resolveTables({
|
|
@@ -481,7 +482,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
481
482
|
{ storageVersion }
|
|
482
483
|
)
|
|
483
484
|
);
|
|
484
|
-
const bucketStorage = factory
|
|
485
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
485
486
|
|
|
486
487
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
487
488
|
const resolved = await writer.resolveTables({
|
|
@@ -550,7 +551,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
550
551
|
// Here we're persisting one sync config, then resolving tables with others.
|
|
551
552
|
// We're also using the default hydration state for them all.
|
|
552
553
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
|
|
553
|
-
const bucketStorage = factory
|
|
554
|
+
const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
|
|
554
555
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
555
556
|
const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
|
|
556
557
|
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
|
|
@@ -666,7 +667,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
666
667
|
|
|
667
668
|
await using factory = await storageConfig.factory();
|
|
668
669
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
|
|
669
|
-
const bucketStorage = factory
|
|
670
|
+
const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
|
|
670
671
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
671
672
|
const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
|
|
672
673
|
const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
|
|
@@ -739,7 +740,7 @@ event_definitions:
|
|
|
739
740
|
});
|
|
740
741
|
try {
|
|
741
742
|
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion }));
|
|
742
|
-
const firstStorage = factory
|
|
743
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
743
744
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
744
745
|
const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
|
|
745
746
|
const firstResolved = await firstWriter.resolveTables({
|
|
@@ -755,7 +756,7 @@ event_definitions:
|
|
|
755
756
|
|
|
756
757
|
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(secondYaml, { storageVersion }));
|
|
757
758
|
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
758
|
-
const secondStorage = factory
|
|
759
|
+
const secondStorage = (await getTestStorage(factory, second)) as MongoSyncBucketStorage;
|
|
759
760
|
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
760
761
|
const resolved = await secondWriter.resolveTables({
|
|
761
762
|
connection_id: 1,
|
|
@@ -806,7 +807,7 @@ event_definitions:
|
|
|
806
807
|
});
|
|
807
808
|
try {
|
|
808
809
|
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion }));
|
|
809
|
-
const firstStorage = factory
|
|
810
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
810
811
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
811
812
|
const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
|
|
812
813
|
const firstResolved = await firstWriter.resolveTables({
|
|
@@ -821,7 +822,7 @@ event_definitions:
|
|
|
821
822
|
|
|
822
823
|
const second = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(false), { storageVersion }));
|
|
823
824
|
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
824
|
-
const secondStorage = factory
|
|
825
|
+
const secondStorage = (await getTestStorage(factory, second)) as MongoSyncBucketStorage;
|
|
825
826
|
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
826
827
|
const resolved = await secondWriter.resolveTables({
|
|
827
828
|
connection_id: 1,
|
|
@@ -875,7 +876,7 @@ event_definitions:
|
|
|
875
876
|
{ storageVersion }
|
|
876
877
|
)
|
|
877
878
|
);
|
|
878
|
-
const bucketStorage = factory
|
|
879
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
879
880
|
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
880
881
|
const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
|
|
881
882
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -988,7 +989,7 @@ streams:
|
|
|
988
989
|
expect(parsed.syncConfigs).toHaveLength(1);
|
|
989
990
|
expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
|
|
990
991
|
|
|
991
|
-
const bucketStorage = factory
|
|
992
|
+
const bucketStorage = await getTestStorage(factory, replicatingStreams[0]);
|
|
992
993
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
993
994
|
const resolved = await writer.resolveTables({
|
|
994
995
|
connection_id: 1,
|
|
@@ -1015,7 +1016,7 @@ streams:
|
|
|
1015
1016
|
{ storageVersion }
|
|
1016
1017
|
)
|
|
1017
1018
|
);
|
|
1018
|
-
const firstStorage = factory
|
|
1019
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1019
1020
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1020
1021
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1021
1022
|
await firstWriter.commit('1/1');
|
|
@@ -1035,7 +1036,7 @@ streams:
|
|
|
1035
1036
|
);
|
|
1036
1037
|
expect(second.replicationStreamId).toEqual(first.replicationStreamId);
|
|
1037
1038
|
|
|
1038
|
-
const bucketStorage = factory
|
|
1039
|
+
const bucketStorage = await getTestStorage(factory, second);
|
|
1039
1040
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1040
1041
|
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
1041
1042
|
|
|
@@ -1050,7 +1051,10 @@ streams:
|
|
|
1050
1051
|
await writer.markAllSnapshotDone('2/1');
|
|
1051
1052
|
await writer.commit('2/1');
|
|
1052
1053
|
|
|
1053
|
-
const activeStorage = (await
|
|
1054
|
+
const activeStorage = (await getTestStorage(
|
|
1055
|
+
factory,
|
|
1056
|
+
(await factory.getActiveSyncConfig())!.replicationStream
|
|
1057
|
+
)) as MongoSyncBucketStorage;
|
|
1054
1058
|
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1055
1059
|
const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
|
|
1056
1060
|
expect(activeStatus?.bucketDataSources).toHaveLength(1);
|
|
@@ -1074,7 +1078,7 @@ streams:
|
|
|
1074
1078
|
{ storageVersion }
|
|
1075
1079
|
)
|
|
1076
1080
|
);
|
|
1077
|
-
const firstStorage = factory
|
|
1081
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1078
1082
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1079
1083
|
const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
|
|
1080
1084
|
const firstResolved = await firstWriter.resolveTables({
|
|
@@ -1103,7 +1107,7 @@ streams:
|
|
|
1103
1107
|
|
|
1104
1108
|
const deploying = await factory.getDeployingSyncConfig();
|
|
1105
1109
|
expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
|
|
1106
|
-
const bucketStorage = deploying!.
|
|
1110
|
+
const bucketStorage = await getTestStorage(factory, deploying!.replicationStream);
|
|
1107
1111
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1108
1112
|
const resolved = await writer.resolveTables({
|
|
1109
1113
|
connection_id: 1,
|
|
@@ -1149,7 +1153,7 @@ streams:
|
|
|
1149
1153
|
`;
|
|
1150
1154
|
|
|
1151
1155
|
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
|
|
1152
|
-
const firstStorage = factory
|
|
1156
|
+
const firstStorage = await getTestStorage(factory, first);
|
|
1153
1157
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1154
1158
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1155
1159
|
await firstWriter.commit('1/1');
|
|
@@ -1201,7 +1205,7 @@ streams:
|
|
|
1201
1205
|
{ storageVersion }
|
|
1202
1206
|
)
|
|
1203
1207
|
);
|
|
1204
|
-
const firstStorage = factory
|
|
1208
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1205
1209
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1206
1210
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1207
1211
|
await firstWriter.commit('1/1');
|
|
@@ -1223,7 +1227,7 @@ streams:
|
|
|
1223
1227
|
);
|
|
1224
1228
|
expect(second.replicationStreamId).toBe(first.replicationStreamId);
|
|
1225
1229
|
|
|
1226
|
-
const bucketStorage = factory
|
|
1230
|
+
const bucketStorage = await getTestStorage(factory, second);
|
|
1227
1231
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1228
1232
|
const resolved = await writer.resolveTables({
|
|
1229
1233
|
connection_id: 1,
|
|
@@ -1274,7 +1278,7 @@ streams:
|
|
|
1274
1278
|
{ storageVersion }
|
|
1275
1279
|
)
|
|
1276
1280
|
);
|
|
1277
|
-
const firstStorage = factory
|
|
1281
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1278
1282
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1279
1283
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1280
1284
|
await firstWriter.commit('1/1');
|
|
@@ -1317,7 +1321,7 @@ streams:
|
|
|
1317
1321
|
expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
|
|
1318
1322
|
}
|
|
1319
1323
|
|
|
1320
|
-
const secondStorage = factory
|
|
1324
|
+
const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorage;
|
|
1321
1325
|
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1322
1326
|
await secondWriter.markAllSnapshotDone('2/1');
|
|
1323
1327
|
await secondWriter.commit('2/1');
|
|
@@ -1360,7 +1364,7 @@ streams:
|
|
|
1360
1364
|
{ storageVersion }
|
|
1361
1365
|
)
|
|
1362
1366
|
);
|
|
1363
|
-
const activeStorage = factory
|
|
1367
|
+
const activeStorage = (await getTestStorage(factory, active)) as MongoSyncBucketStorage;
|
|
1364
1368
|
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1365
1369
|
await activeWriter.markAllSnapshotDone('1/1');
|
|
1366
1370
|
await activeWriter.commit('1/1');
|
|
@@ -1396,7 +1400,7 @@ streams:
|
|
|
1396
1400
|
{ storageVersion }
|
|
1397
1401
|
)
|
|
1398
1402
|
);
|
|
1399
|
-
const activeStorage = factory
|
|
1403
|
+
const activeStorage = (await getTestStorage(factory, active)) as MongoSyncBucketStorage;
|
|
1400
1404
|
await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1401
1405
|
await activeWriter.markAllSnapshotDone('1/1');
|
|
1402
1406
|
await activeWriter.commit('1/1');
|
|
@@ -1455,7 +1459,7 @@ streams:
|
|
|
1455
1459
|
{ storageVersion }
|
|
1456
1460
|
)
|
|
1457
1461
|
);
|
|
1458
|
-
const firstStorage = factory
|
|
1462
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1459
1463
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1460
1464
|
const sourceTable = await test_utils.resolveTestTable(
|
|
1461
1465
|
firstWriter,
|
|
@@ -1505,7 +1509,7 @@ streams:
|
|
|
1505
1509
|
|
|
1506
1510
|
const replicatingStreams = await factory.getReplicatingReplicationStreams();
|
|
1507
1511
|
expect(replicatingStreams).toHaveLength(1);
|
|
1508
|
-
const secondStorage = factory
|
|
1512
|
+
const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorage;
|
|
1509
1513
|
await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1510
1514
|
// No new data replicated - just complete the snapshot and commit.
|
|
1511
1515
|
await secondWriter.markAllSnapshotDone('2/1');
|
|
@@ -1524,7 +1528,10 @@ streams:
|
|
|
1524
1528
|
expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1525
1529
|
|
|
1526
1530
|
// After activation, the active config's checkpoint does not regress.
|
|
1527
|
-
const activeStorage = (await
|
|
1531
|
+
const activeStorage = (await getTestStorage(
|
|
1532
|
+
factory,
|
|
1533
|
+
(await factory.getActiveSyncConfig())!.replicationStream
|
|
1534
|
+
)) as MongoSyncBucketStorage;
|
|
1528
1535
|
const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
|
|
1529
1536
|
expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
|
|
1530
1537
|
}
|
|
@@ -1546,7 +1553,7 @@ streams:
|
|
|
1546
1553
|
{ storageVersion }
|
|
1547
1554
|
)
|
|
1548
1555
|
);
|
|
1549
|
-
const firstStorage = factory
|
|
1556
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1550
1557
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1551
1558
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1552
1559
|
await firstWriter.commit('1/1');
|
|
@@ -1593,7 +1600,7 @@ streams:
|
|
|
1593
1600
|
{ storageVersion }
|
|
1594
1601
|
)
|
|
1595
1602
|
);
|
|
1596
|
-
const firstStorage = factory
|
|
1603
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1597
1604
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1598
1605
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1599
1606
|
await firstWriter.commit('1/1');
|
|
@@ -1608,7 +1615,7 @@ streams:
|
|
|
1608
1615
|
await using factory = await storageConfig.factory();
|
|
1609
1616
|
|
|
1610
1617
|
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
1611
|
-
const firstStorage = factory
|
|
1618
|
+
const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage;
|
|
1612
1619
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1613
1620
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1614
1621
|
await firstWriter.commit('1/1');
|
|
@@ -1681,7 +1688,7 @@ streams:
|
|
|
1681
1688
|
const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstRules, { storageVersion, lock: true }));
|
|
1682
1689
|
expect(first.current_lock?.sync_rules_id).toBe(first.replicationStreamId);
|
|
1683
1690
|
try {
|
|
1684
|
-
const firstStorage = factory.getInstance(first);
|
|
1691
|
+
const firstStorage = factory.getInstance(first, { replicationLock: first.current_lock! });
|
|
1685
1692
|
await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1686
1693
|
await firstWriter.markAllSnapshotDone('1/1');
|
|
1687
1694
|
await firstWriter.commit('1/1');
|
|
@@ -1709,7 +1716,7 @@ streams:
|
|
|
1709
1716
|
{ storageVersion }
|
|
1710
1717
|
)
|
|
1711
1718
|
);
|
|
1712
|
-
const bucketStorage = factory
|
|
1719
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
1713
1720
|
|
|
1714
1721
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1715
1722
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -1748,7 +1755,7 @@ streams:
|
|
|
1748
1755
|
{ storageVersion }
|
|
1749
1756
|
)
|
|
1750
1757
|
);
|
|
1751
|
-
const bucketStorage = factory
|
|
1758
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
1752
1759
|
|
|
1753
1760
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1754
1761
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -1782,7 +1789,7 @@ streams:
|
|
|
1782
1789
|
storageVersion
|
|
1783
1790
|
})
|
|
1784
1791
|
);
|
|
1785
|
-
const bucketStorage = factory
|
|
1792
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
1786
1793
|
const mongoFactory = factory as MongoBucketStorage;
|
|
1787
1794
|
const sourceTableId = new bson.ObjectId();
|
|
1788
1795
|
const documents = Array.from({ length: 10_002 }, (_, index) => ({
|
|
@@ -1825,7 +1832,7 @@ streams:
|
|
|
1825
1832
|
storageVersion
|
|
1826
1833
|
})
|
|
1827
1834
|
);
|
|
1828
|
-
const bucketStorage = factory
|
|
1835
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
1829
1836
|
const mongoFactory = factory as MongoBucketStorage;
|
|
1830
1837
|
const sourceTableId = new bson.ObjectId();
|
|
1831
1838
|
const documents = Array.from({ length: 10_002 }, (_, index) => ({
|
|
@@ -1876,7 +1883,7 @@ streams:
|
|
|
1876
1883
|
{ storageVersion }
|
|
1877
1884
|
)
|
|
1878
1885
|
);
|
|
1879
|
-
const bucketStorage = factory
|
|
1886
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
1880
1887
|
const metricsBefore = await factory.getStorageMetrics();
|
|
1881
1888
|
|
|
1882
1889
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -1939,7 +1946,7 @@ streams:
|
|
|
1939
1946
|
{ storageVersion }
|
|
1940
1947
|
)
|
|
1941
1948
|
);
|
|
1942
|
-
const bucketStorage = factory
|
|
1949
|
+
const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
|
|
1943
1950
|
const previousCheckpoint = await bucketStorage.getCheckpoint();
|
|
1944
1951
|
|
|
1945
1952
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -1984,7 +1991,7 @@ streams:
|
|
|
1984
1991
|
);
|
|
1985
1992
|
|
|
1986
1993
|
const mongoFactory = factory as MongoBucketStorage;
|
|
1987
|
-
const bucketStorage = mongoFactory
|
|
1994
|
+
const bucketStorage = await getTestStorage(mongoFactory, syncRules);
|
|
1988
1995
|
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1989
1996
|
await db.initializeStreamStorage(syncRules.replicationStreamId);
|
|
1990
1997
|
|
|
@@ -2079,7 +2086,7 @@ describe('sync - mongodb', () => {
|
|
|
2079
2086
|
const legacySyncRules = await factory.updateSyncRules(
|
|
2080
2087
|
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.LEGACY_STORAGE_VERSION })
|
|
2081
2088
|
);
|
|
2082
|
-
const legacyStorage = factory
|
|
2089
|
+
const legacyStorage = await getTestStorage(factory, legacySyncRules);
|
|
2083
2090
|
await using legacyWriter = await legacyStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
2084
2091
|
await legacyWriter.markAllSnapshotDone('1/1');
|
|
2085
2092
|
await legacyWriter.commit('1/1');
|
|
@@ -2091,7 +2098,7 @@ describe('sync - mongodb', () => {
|
|
|
2091
2098
|
const v3SyncRules = await factory.updateSyncRules(
|
|
2092
2099
|
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.STORAGE_VERSION_3 })
|
|
2093
2100
|
);
|
|
2094
|
-
const v3Storage = factory
|
|
2101
|
+
const v3Storage = await getTestStorage(factory, v3SyncRules);
|
|
2095
2102
|
await using v3Writer = await v3Storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
2096
2103
|
await v3Writer.markAllSnapshotDone('2/1');
|
|
2097
2104
|
await v3Writer.commit('2/1');
|
|
@@ -2119,7 +2126,7 @@ describe('sync - mongodb', () => {
|
|
|
2119
2126
|
{ storageVersion }
|
|
2120
2127
|
)
|
|
2121
2128
|
);
|
|
2122
|
-
const bucketStorage = factory
|
|
2129
|
+
const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
|
|
2123
2130
|
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
2124
2131
|
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
2125
2132
|
|
|
@@ -2251,7 +2258,7 @@ describe('sync - mongodb', () => {
|
|
|
2251
2258
|
{ storageVersion }
|
|
2252
2259
|
)
|
|
2253
2260
|
);
|
|
2254
|
-
const bucketStorage = factory
|
|
2261
|
+
const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorageV3;
|
|
2255
2262
|
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
2256
2263
|
|
|
2257
2264
|
const start = 5n;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
2
|
-
import { bucketRequest, test_utils } from '@powersync/service-core-tests';
|
|
2
|
+
import { bucketRequest, getTestStorage, test_utils } from '@powersync/service-core-tests';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
import { describe, expect, test } from 'vitest';
|
|
5
5
|
import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
|
|
@@ -80,7 +80,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
80
80
|
test('resolveTables derives storeCurrentData fresh each call, with no persisted memory', async () => {
|
|
81
81
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
82
82
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
83
|
-
const bucketStorage = factory
|
|
83
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
84
84
|
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
85
85
|
|
|
86
86
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -114,7 +114,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
114
114
|
test('storeCurrentData=false omits the row payload from current_data, data still syncs', async () => {
|
|
115
115
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
116
116
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
117
|
-
const bucketStorage = factory
|
|
117
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
118
118
|
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
119
119
|
|
|
120
120
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -146,7 +146,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
146
146
|
test('storeCurrentData=true retains the row payload in current_data', async () => {
|
|
147
147
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
148
148
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
149
|
-
const bucketStorage = factory
|
|
149
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
150
150
|
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
151
151
|
|
|
152
152
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -179,7 +179,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
179
179
|
test('storeCurrentData=false processes UPDATE without a stored copy', async () => {
|
|
180
180
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
181
181
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
182
|
-
const bucketStorage = factory
|
|
182
|
+
const bucketStorage = await getTestStorage(factory, syncRules);
|
|
183
183
|
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
184
184
|
|
|
185
185
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|