@powersync/service-core-tests 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/test-utils/general-utils.d.ts +0 -1
- package/dist/test-utils/general-utils.js +0 -24
- package/dist/test-utils/general-utils.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +40 -51
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +11 -26
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +80 -96
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +84 -46
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +5 -9
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.d.ts +3 -1
- package/dist/tests/register-sync-tests.js +29 -22
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/dist/tests/util.d.ts +5 -1
- package/dist/tests/util.js +14 -0
- package/dist/tests/util.js.map +1 -1
- package/package.json +3 -3
- package/src/test-utils/general-utils.ts +1 -25
- package/src/tests/register-compacting-tests.ts +56 -51
- package/src/tests/register-data-storage-checkpoint-tests.ts +21 -26
- package/src/tests/register-data-storage-data-tests.ts +135 -101
- package/src/tests/register-data-storage-parameter-tests.ts +102 -47
- package/src/tests/register-parameter-compacting-tests.ts +9 -9
- package/src/tests/register-sync-tests.ts +32 -22
- package/src/tests/util.ts +21 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -50,10 +50,10 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
-
import { getUuidReplicaIdentityBson, storage } from '@powersync/service-core';
|
|
53
|
+
import { getUuidReplicaIdentityBson, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
54
54
|
import { describe, expect, test } from 'vitest';
|
|
55
55
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
56
|
-
import { TEST_TABLE } from './util.js';
|
|
56
|
+
import { bucketRequest, bucketRequestMap, bucketRequests, TEST_TABLE } from './util.js';
|
|
57
57
|
/**
|
|
58
58
|
* Normalize data from OplogEntries for comparison in tests.
|
|
59
59
|
* Tests typically expect the stringified result
|
|
@@ -79,14 +79,12 @@ export function registerDataStorageDataTests(generateStorageFactory) {
|
|
|
79
79
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
80
80
|
try {
|
|
81
81
|
const factory = __addDisposableResource(env_1, await generateStorageFactory(), true);
|
|
82
|
-
const syncRules = await factory.updateSyncRules(
|
|
83
|
-
content: `
|
|
82
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
84
83
|
bucket_definitions:
|
|
85
84
|
global:
|
|
86
85
|
data:
|
|
87
86
|
- SELECT id, description FROM "%"
|
|
88
|
-
`
|
|
89
|
-
});
|
|
87
|
+
`));
|
|
90
88
|
const bucketStorage = factory.getInstance(syncRules);
|
|
91
89
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
92
90
|
const sourceTable = TEST_TABLE;
|
|
@@ -107,7 +105,7 @@ bucket_definitions:
|
|
|
107
105
|
await batch.commit('1/1');
|
|
108
106
|
});
|
|
109
107
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
110
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
108
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
|
|
111
109
|
const data = batch[0].chunkData.data.map((d) => {
|
|
112
110
|
return {
|
|
113
111
|
op: d.op,
|
|
@@ -121,10 +119,12 @@ bucket_definitions:
|
|
|
121
119
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
122
120
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
123
121
|
]);
|
|
124
|
-
const checksums = [
|
|
122
|
+
const checksums = [
|
|
123
|
+
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
124
|
+
];
|
|
125
125
|
expect(checksums).toEqual([
|
|
126
126
|
{
|
|
127
|
-
bucket: 'global[]',
|
|
127
|
+
bucket: bucketRequest(syncRules, 'global[]'),
|
|
128
128
|
checksum: (c1 + c2) & 0xffffffff,
|
|
129
129
|
count: 2
|
|
130
130
|
}
|
|
@@ -144,14 +144,12 @@ bucket_definitions:
|
|
|
144
144
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
145
145
|
try {
|
|
146
146
|
const factory = __addDisposableResource(env_2, await generateStorageFactory(), true);
|
|
147
|
-
const syncRules = await factory.updateSyncRules(
|
|
148
|
-
content: `
|
|
147
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
149
148
|
bucket_definitions:
|
|
150
149
|
global:
|
|
151
150
|
data:
|
|
152
151
|
- SELECT client_id as id, description FROM "%"
|
|
153
|
-
`
|
|
154
|
-
});
|
|
152
|
+
`));
|
|
155
153
|
const bucketStorage = factory.getInstance(syncRules);
|
|
156
154
|
const sourceTable = TEST_TABLE;
|
|
157
155
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
@@ -188,7 +186,7 @@ bucket_definitions:
|
|
|
188
186
|
await batch.commit('1/1');
|
|
189
187
|
});
|
|
190
188
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
191
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
189
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
|
|
192
190
|
const data = batch[0].chunkData.data.map((d) => {
|
|
193
191
|
return {
|
|
194
192
|
op: d.op,
|
|
@@ -216,14 +214,12 @@ bucket_definitions:
|
|
|
216
214
|
const env_3 = { stack: [], error: void 0, hasError: false };
|
|
217
215
|
try {
|
|
218
216
|
const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
|
|
219
|
-
const syncRules = await factory.updateSyncRules(
|
|
220
|
-
content: `
|
|
217
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
221
218
|
bucket_definitions:
|
|
222
219
|
global:
|
|
223
220
|
data:
|
|
224
221
|
- SELECT id, description FROM "%"
|
|
225
|
-
`
|
|
226
|
-
});
|
|
222
|
+
`));
|
|
227
223
|
const bucketStorage = factory.getInstance(syncRules);
|
|
228
224
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
229
225
|
const sourceTable = TEST_TABLE;
|
|
@@ -255,7 +251,7 @@ bucket_definitions:
|
|
|
255
251
|
});
|
|
256
252
|
});
|
|
257
253
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
258
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
254
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
|
|
259
255
|
const data = batch[0].chunkData.data.map((d) => {
|
|
260
256
|
return {
|
|
261
257
|
op: d.op,
|
|
@@ -269,10 +265,12 @@ bucket_definitions:
|
|
|
269
265
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
270
266
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
271
267
|
]);
|
|
272
|
-
const checksums = [
|
|
268
|
+
const checksums = [
|
|
269
|
+
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
270
|
+
];
|
|
273
271
|
expect(checksums).toEqual([
|
|
274
272
|
{
|
|
275
|
-
bucket: 'global[]',
|
|
273
|
+
bucket: bucketRequest(syncRules, 'global[]'),
|
|
276
274
|
checksum: (c1 + c2) & 0xffffffff,
|
|
277
275
|
count: 2
|
|
278
276
|
}
|
|
@@ -292,14 +290,12 @@ bucket_definitions:
|
|
|
292
290
|
const env_4 = { stack: [], error: void 0, hasError: false };
|
|
293
291
|
try {
|
|
294
292
|
const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
|
|
295
|
-
const syncRules = await factory.updateSyncRules(
|
|
296
|
-
content: `
|
|
293
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
297
294
|
bucket_definitions:
|
|
298
295
|
global:
|
|
299
296
|
data:
|
|
300
297
|
- SELECT id, description FROM "%"
|
|
301
|
-
`
|
|
302
|
-
});
|
|
298
|
+
`));
|
|
303
299
|
const bucketStorage = factory.getInstance(syncRules);
|
|
304
300
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
305
301
|
const sourceTable = TEST_TABLE;
|
|
@@ -368,7 +364,7 @@ bucket_definitions:
|
|
|
368
364
|
await batch.commit('2/1');
|
|
369
365
|
});
|
|
370
366
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
371
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
367
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
|
|
372
368
|
const data = batch[0].chunkData.data.map((d) => {
|
|
373
369
|
return {
|
|
374
370
|
op: d.op,
|
|
@@ -384,10 +380,12 @@ bucket_definitions:
|
|
|
384
380
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
385
381
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
386
382
|
]);
|
|
387
|
-
const checksums = [
|
|
383
|
+
const checksums = [
|
|
384
|
+
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
385
|
+
];
|
|
388
386
|
expect(checksums).toEqual([
|
|
389
387
|
{
|
|
390
|
-
bucket: 'global[]',
|
|
388
|
+
bucket: bucketRequest(syncRules, 'global[]'),
|
|
391
389
|
checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
|
|
392
390
|
count: 4
|
|
393
391
|
}
|
|
@@ -414,14 +412,12 @@ bucket_definitions:
|
|
|
414
412
|
// 1. Not getting the correct "current_data" state for each operation.
|
|
415
413
|
// 2. Output order not being correct.
|
|
416
414
|
const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
|
|
417
|
-
const syncRules = await factory.updateSyncRules(
|
|
418
|
-
content: `
|
|
415
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
419
416
|
bucket_definitions:
|
|
420
417
|
global:
|
|
421
418
|
data:
|
|
422
419
|
- SELECT id, description FROM "test"
|
|
423
|
-
`
|
|
424
|
-
});
|
|
420
|
+
`));
|
|
425
421
|
const bucketStorage = factory.getInstance(syncRules);
|
|
426
422
|
// Pre-setup
|
|
427
423
|
const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
@@ -520,7 +516,7 @@ bucket_definitions:
|
|
|
520
516
|
});
|
|
521
517
|
});
|
|
522
518
|
const checkpoint2 = result2.flushed_op;
|
|
523
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2,
|
|
519
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', checkpoint1]])));
|
|
524
520
|
const data = batch[0].chunkData.data.map((d) => {
|
|
525
521
|
return {
|
|
526
522
|
op: d.op,
|
|
@@ -564,14 +560,12 @@ bucket_definitions:
|
|
|
564
560
|
]);
|
|
565
561
|
}
|
|
566
562
|
const factory = __addDisposableResource(env_6, await generateStorageFactory(), true);
|
|
567
|
-
const syncRules = await factory.updateSyncRules(
|
|
568
|
-
content: `
|
|
563
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
569
564
|
bucket_definitions:
|
|
570
565
|
global:
|
|
571
566
|
data:
|
|
572
567
|
- SELECT id, description FROM "test"
|
|
573
|
-
`
|
|
574
|
-
});
|
|
568
|
+
`));
|
|
575
569
|
const bucketStorage = factory.getInstance(syncRules);
|
|
576
570
|
const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
|
|
577
571
|
// Pre-setup
|
|
@@ -618,7 +612,7 @@ bucket_definitions:
|
|
|
618
612
|
});
|
|
619
613
|
});
|
|
620
614
|
const checkpoint3 = result3.flushed_op;
|
|
621
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3,
|
|
615
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])));
|
|
622
616
|
const data = batch[0].chunkData.data.map((d) => {
|
|
623
617
|
return {
|
|
624
618
|
op: d.op,
|
|
@@ -672,14 +666,12 @@ bucket_definitions:
|
|
|
672
666
|
]);
|
|
673
667
|
}
|
|
674
668
|
const factory = __addDisposableResource(env_7, await generateStorageFactory(), true);
|
|
675
|
-
const syncRules = await factory.updateSyncRules(
|
|
676
|
-
content: `
|
|
669
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
677
670
|
bucket_definitions:
|
|
678
671
|
global:
|
|
679
672
|
data:
|
|
680
673
|
- SELECT id, description FROM "test"
|
|
681
|
-
`
|
|
682
|
-
});
|
|
674
|
+
`));
|
|
683
675
|
const bucketStorage = factory.getInstance(syncRules);
|
|
684
676
|
const sourceTable = test_utils.makeTestTable('test', ['id', 'description']);
|
|
685
677
|
// Pre-setup
|
|
@@ -726,7 +718,7 @@ bucket_definitions:
|
|
|
726
718
|
});
|
|
727
719
|
});
|
|
728
720
|
const checkpoint3 = result3.flushed_op;
|
|
729
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3,
|
|
721
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])));
|
|
730
722
|
const data = batch[0].chunkData.data.map((d) => {
|
|
731
723
|
return {
|
|
732
724
|
op: d.op,
|
|
@@ -771,14 +763,12 @@ bucket_definitions:
|
|
|
771
763
|
// The specific batch splits is an implementation detail of the storage driver,
|
|
772
764
|
// and the test will have to updated when other implementations are added.
|
|
773
765
|
const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
|
|
774
|
-
const syncRules = await factory.updateSyncRules(
|
|
775
|
-
content: `
|
|
766
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
776
767
|
bucket_definitions:
|
|
777
768
|
global:
|
|
778
769
|
data:
|
|
779
770
|
- SELECT id, description FROM "%"
|
|
780
|
-
`
|
|
781
|
-
});
|
|
771
|
+
`));
|
|
782
772
|
const bucketStorage = factory.getInstance(syncRules);
|
|
783
773
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
784
774
|
const sourceTable = TEST_TABLE;
|
|
@@ -826,7 +816,7 @@ bucket_definitions:
|
|
|
826
816
|
const options = {
|
|
827
817
|
chunkLimitBytes: 16 * 1024 * 1024
|
|
828
818
|
};
|
|
829
|
-
const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
819
|
+
const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options));
|
|
830
820
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
831
821
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
832
822
|
{ op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
|
|
@@ -836,7 +826,7 @@ bucket_definitions:
|
|
|
836
826
|
has_more: true,
|
|
837
827
|
next_after: '2'
|
|
838
828
|
});
|
|
839
|
-
const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
829
|
+
const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]), options));
|
|
840
830
|
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
841
831
|
{ op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
|
|
842
832
|
{ op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
|
|
@@ -846,7 +836,7 @@ bucket_definitions:
|
|
|
846
836
|
has_more: false,
|
|
847
837
|
next_after: '4'
|
|
848
838
|
});
|
|
849
|
-
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
839
|
+
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]), options));
|
|
850
840
|
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
851
841
|
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
852
842
|
}
|
|
@@ -865,14 +855,12 @@ bucket_definitions:
|
|
|
865
855
|
try {
|
|
866
856
|
// Test syncing a batch of data that is limited by count.
|
|
867
857
|
const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
|
|
868
|
-
const syncRules = await factory.updateSyncRules(
|
|
869
|
-
content: `
|
|
858
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
870
859
|
bucket_definitions:
|
|
871
860
|
global:
|
|
872
861
|
data:
|
|
873
862
|
- SELECT id, description FROM "%"
|
|
874
|
-
`
|
|
875
|
-
});
|
|
863
|
+
`));
|
|
876
864
|
const bucketStorage = factory.getInstance(syncRules);
|
|
877
865
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
878
866
|
const sourceTable = TEST_TABLE;
|
|
@@ -890,7 +878,7 @@ bucket_definitions:
|
|
|
890
878
|
await batch.commit('1/1');
|
|
891
879
|
});
|
|
892
880
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
893
|
-
const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
881
|
+
const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), { limit: 4 }));
|
|
894
882
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
895
883
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
896
884
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
@@ -902,7 +890,7 @@ bucket_definitions:
|
|
|
902
890
|
has_more: true,
|
|
903
891
|
next_after: '4'
|
|
904
892
|
});
|
|
905
|
-
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
893
|
+
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch1.chunkData.next_after)]]), {
|
|
906
894
|
limit: 4
|
|
907
895
|
}));
|
|
908
896
|
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
@@ -914,7 +902,7 @@ bucket_definitions:
|
|
|
914
902
|
has_more: false,
|
|
915
903
|
next_after: '6'
|
|
916
904
|
});
|
|
917
|
-
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
905
|
+
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch2.chunkData.next_after)]]), {
|
|
918
906
|
limit: 4
|
|
919
907
|
}));
|
|
920
908
|
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
@@ -935,8 +923,7 @@ bucket_definitions:
|
|
|
935
923
|
const env_10 = { stack: [], error: void 0, hasError: false };
|
|
936
924
|
try {
|
|
937
925
|
const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
|
|
938
|
-
const syncRules = await factory.updateSyncRules(
|
|
939
|
-
content: `
|
|
926
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
940
927
|
bucket_definitions:
|
|
941
928
|
global1:
|
|
942
929
|
data:
|
|
@@ -944,8 +931,7 @@ bucket_definitions:
|
|
|
944
931
|
global2:
|
|
945
932
|
data:
|
|
946
933
|
- SELECT id, description FROM test WHERE bucket = 'global2'
|
|
947
|
-
`
|
|
948
|
-
});
|
|
934
|
+
`));
|
|
949
935
|
const bucketStorage = factory.getInstance(syncRules);
|
|
950
936
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
951
937
|
const sourceTable = TEST_TABLE;
|
|
@@ -964,10 +950,11 @@ bucket_definitions:
|
|
|
964
950
|
await batch.commit('1/1');
|
|
965
951
|
});
|
|
966
952
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
967
|
-
|
|
953
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [
|
|
968
954
|
['global1[]', 0n],
|
|
969
955
|
['global2[]', 0n]
|
|
970
956
|
]), options));
|
|
957
|
+
return { syncRules, batch };
|
|
971
958
|
}
|
|
972
959
|
catch (e_10) {
|
|
973
960
|
env_10.error = e_10;
|
|
@@ -980,10 +967,10 @@ bucket_definitions:
|
|
|
980
967
|
}
|
|
981
968
|
};
|
|
982
969
|
test('batch has_more (1)', async () => {
|
|
983
|
-
const batch = await setup({ limit: 5 });
|
|
970
|
+
const { batch, syncRules } = await setup({ limit: 5 });
|
|
984
971
|
expect(batch.length).toEqual(2);
|
|
985
|
-
expect(batch[0].chunkData.bucket).toEqual('global1[]');
|
|
986
|
-
expect(batch[1].chunkData.bucket).toEqual('global2[]');
|
|
972
|
+
expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
|
|
973
|
+
expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
|
|
987
974
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
988
975
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
989
976
|
]);
|
|
@@ -1005,10 +992,10 @@ bucket_definitions:
|
|
|
1005
992
|
});
|
|
1006
993
|
});
|
|
1007
994
|
test('batch has_more (2)', async () => {
|
|
1008
|
-
const batch = await setup({ limit: 11 });
|
|
995
|
+
const { batch, syncRules } = await setup({ limit: 11 });
|
|
1009
996
|
expect(batch.length).toEqual(2);
|
|
1010
|
-
expect(batch[0].chunkData.bucket).toEqual('global1[]');
|
|
1011
|
-
expect(batch[1].chunkData.bucket).toEqual('global2[]');
|
|
997
|
+
expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
|
|
998
|
+
expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
|
|
1012
999
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1013
1000
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1014
1001
|
]);
|
|
@@ -1036,11 +1023,11 @@ bucket_definitions:
|
|
|
1036
1023
|
});
|
|
1037
1024
|
test('batch has_more (3)', async () => {
|
|
1038
1025
|
// 50 bytes is more than 1 row, less than 2 rows
|
|
1039
|
-
const batch = await setup({ limit: 3, chunkLimitBytes: 50 });
|
|
1026
|
+
const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 });
|
|
1040
1027
|
expect(batch.length).toEqual(3);
|
|
1041
|
-
expect(batch[0].chunkData.bucket).toEqual('global1[]');
|
|
1042
|
-
expect(batch[1].chunkData.bucket).toEqual('global2[]');
|
|
1043
|
-
expect(batch[2].chunkData.bucket).toEqual('global2[]');
|
|
1028
|
+
expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]'));
|
|
1029
|
+
expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
|
|
1030
|
+
expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]'));
|
|
1044
1031
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1045
1032
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1046
1033
|
]);
|
|
@@ -1077,13 +1064,12 @@ bucket_definitions:
|
|
|
1077
1064
|
parameters_size_bytes: 0,
|
|
1078
1065
|
replication_size_bytes: 0
|
|
1079
1066
|
});
|
|
1080
|
-
const r = await f.configureSyncRules(
|
|
1067
|
+
const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
|
|
1081
1068
|
const storage = f.getInstance(r.persisted_sync_rules);
|
|
1082
1069
|
await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
1083
1070
|
await batch.keepalive('1/0');
|
|
1084
1071
|
});
|
|
1085
|
-
|
|
1086
|
-
expect(metrics2).toMatchSnapshot();
|
|
1072
|
+
await f.getStorageMetrics();
|
|
1087
1073
|
}
|
|
1088
1074
|
catch (e_11) {
|
|
1089
1075
|
env_11.error = e_11;
|
|
@@ -1102,15 +1088,13 @@ bucket_definitions:
|
|
|
1102
1088
|
// but large enough in size to be split over multiple returned chunks.
|
|
1103
1089
|
// Similar to the above test, but splits over 1MB chunks.
|
|
1104
1090
|
const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
|
|
1105
|
-
const syncRules = await factory.updateSyncRules(
|
|
1106
|
-
content: `
|
|
1091
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1107
1092
|
bucket_definitions:
|
|
1108
1093
|
global:
|
|
1109
1094
|
data:
|
|
1110
1095
|
- SELECT id FROM test
|
|
1111
1096
|
- SELECT id FROM test_ignore WHERE false
|
|
1112
|
-
`
|
|
1113
|
-
});
|
|
1097
|
+
`));
|
|
1114
1098
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1115
1099
|
const sourceTable = test_utils.makeTestTable('test', ['id']);
|
|
1116
1100
|
const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id']);
|
|
@@ -1155,14 +1139,12 @@ bucket_definitions:
|
|
|
1155
1139
|
const env_13 = { stack: [], error: void 0, hasError: false };
|
|
1156
1140
|
try {
|
|
1157
1141
|
const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
|
|
1158
|
-
const syncRules = await factory.updateSyncRules(
|
|
1159
|
-
content: `
|
|
1142
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1160
1143
|
bucket_definitions:
|
|
1161
1144
|
global:
|
|
1162
1145
|
data:
|
|
1163
1146
|
- SELECT client_id as id, description FROM "%"
|
|
1164
|
-
`
|
|
1165
|
-
});
|
|
1147
|
+
`));
|
|
1166
1148
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1167
1149
|
const sourceTable = TEST_TABLE;
|
|
1168
1150
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
@@ -1178,10 +1160,14 @@ bucket_definitions:
|
|
|
1178
1160
|
await batch.commit('1/1');
|
|
1179
1161
|
});
|
|
1180
1162
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1181
|
-
const checksums = [
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
expect(
|
|
1163
|
+
const checksums = [
|
|
1164
|
+
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
1165
|
+
];
|
|
1166
|
+
expect(checksums).toEqual([{ bucket: bucketRequest(syncRules, 'global[]'), checksum: 1917136889, count: 1 }]);
|
|
1167
|
+
const checksums2 = [
|
|
1168
|
+
...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
|
|
1169
|
+
];
|
|
1170
|
+
expect(checksums2).toEqual([{ bucket: bucketRequest(syncRules, 'global[]'), checksum: 1917136889, count: 1 }]);
|
|
1185
1171
|
}
|
|
1186
1172
|
catch (e_13) {
|
|
1187
1173
|
env_13.error = e_13;
|
|
@@ -1205,15 +1191,13 @@ export function testChecksumBatching(generateStorageFactory) {
|
|
|
1205
1191
|
const env_14 = { stack: [], error: void 0, hasError: false };
|
|
1206
1192
|
try {
|
|
1207
1193
|
const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
|
|
1208
|
-
const syncRules = await factory.updateSyncRules(
|
|
1209
|
-
content: `
|
|
1194
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1210
1195
|
bucket_definitions:
|
|
1211
1196
|
user:
|
|
1212
1197
|
parameters: select request.user_id() as user_id
|
|
1213
1198
|
data:
|
|
1214
1199
|
- select id, description from test where user_id = bucket.user_id
|
|
1215
|
-
`
|
|
1216
|
-
});
|
|
1200
|
+
`));
|
|
1217
1201
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1218
1202
|
const sourceTable = TEST_TABLE;
|
|
1219
1203
|
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
@@ -1236,14 +1220,14 @@ bucket_definitions:
|
|
|
1236
1220
|
});
|
|
1237
1221
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1238
1222
|
bucketStorage.clearChecksumCache();
|
|
1239
|
-
const buckets = ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]'];
|
|
1223
|
+
const buckets = bucketRequests(syncRules, ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]']);
|
|
1240
1224
|
const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()];
|
|
1241
1225
|
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1242
1226
|
expect(checksums).toEqual([
|
|
1243
|
-
{ bucket: 'user["u1"]', count: 4, checksum: 346204588 },
|
|
1244
|
-
{ bucket: 'user["u2"]', count: 4, checksum: 5261081 },
|
|
1245
|
-
{ bucket: 'user["u3"]', count: 4, checksum: 134760718 },
|
|
1246
|
-
{ bucket: 'user["u4"]', count: 4, checksum: -302639724 }
|
|
1227
|
+
{ bucket: bucketRequest(syncRules, 'user["u1"]'), count: 4, checksum: 346204588 },
|
|
1228
|
+
{ bucket: bucketRequest(syncRules, 'user["u2"]'), count: 4, checksum: 5261081 },
|
|
1229
|
+
{ bucket: bucketRequest(syncRules, 'user["u3"]'), count: 4, checksum: 134760718 },
|
|
1230
|
+
{ bucket: bucketRequest(syncRules, 'user["u4"]'), count: 4, checksum: -302639724 }
|
|
1247
1231
|
]);
|
|
1248
1232
|
}
|
|
1249
1233
|
catch (e_14) {
|