@powersync/service-core-tests 0.17.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/test-utils/AbstractStreamTestContext.d.ts +1 -1
- package/dist/test-utils/AbstractStreamTestContext.js +2 -3
- package/dist/test-utils/AbstractStreamTestContext.js.map +1 -1
- package/dist/test-utils/StorageDataHelpers.d.ts +3 -2
- package/dist/test-utils/StorageDataHelpers.js +40 -2
- package/dist/test-utils/StorageDataHelpers.js.map +1 -1
- package/dist/test-utils/general-utils.d.ts +3 -1
- package/dist/test-utils/general-utils.js +13 -1
- package/dist/test-utils/general-utils.js.map +1 -1
- package/dist/test-utils/stream_utils.d.ts +0 -1
- package/dist/test-utils/stream_utils.js +0 -10
- package/dist/test-utils/stream_utils.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +7 -7
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +96 -3
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +113 -94
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +259 -75
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/package.json +4 -4
- package/src/test-utils/AbstractStreamTestContext.ts +2 -3
- package/src/test-utils/StorageDataHelpers.ts +45 -2
- package/src/test-utils/general-utils.ts +27 -3
- package/src/test-utils/stream_utils.ts +0 -11
- package/src/tests/register-compacting-tests.ts +7 -7
- package/src/tests/register-data-storage-checkpoint-tests.ts +122 -3
- package/src/tests/register-data-storage-data-tests.ts +129 -132
- package/src/tests/register-sync-tests.ts +280 -77
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -52,9 +52,9 @@ bucket_definitions:
|
|
|
52
52
|
beforeReplicaId: test_utils.rid('test1')
|
|
53
53
|
});
|
|
54
54
|
await writer.commit('1/1');
|
|
55
|
-
const
|
|
55
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
56
56
|
const request = bucketRequest(syncRules, 'global[]');
|
|
57
|
-
const batch = await test_utils.
|
|
57
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
58
58
|
const data = batch[0].chunkData.data.map((d) => {
|
|
59
59
|
return {
|
|
60
60
|
op: d.op,
|
|
@@ -68,9 +68,7 @@ bucket_definitions:
|
|
|
68
68
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
69
69
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
70
70
|
]);
|
|
71
|
-
const checksums = [
|
|
72
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
73
|
-
];
|
|
71
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
74
72
|
expect(checksums).toEqual([
|
|
75
73
|
{
|
|
76
74
|
bucket: request.bucket,
|
|
@@ -107,9 +105,9 @@ bucket_definitions:
|
|
|
107
105
|
afterReplicaId: test_utils.rid('test1')
|
|
108
106
|
});
|
|
109
107
|
await writer.commit('2/1');
|
|
110
|
-
const
|
|
108
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
111
109
|
const request = bucketRequest(syncRules, 'global[]');
|
|
112
|
-
const batch = await test_utils.
|
|
110
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
113
111
|
const data = batch[0].chunkData.data.map((d) => {
|
|
114
112
|
return {
|
|
115
113
|
op: d.op,
|
|
@@ -119,9 +117,7 @@ bucket_definitions:
|
|
|
119
117
|
});
|
|
120
118
|
const c1 = 2871785649;
|
|
121
119
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
122
|
-
const checksums = [
|
|
123
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
124
|
-
];
|
|
120
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
125
121
|
expect(checksums).toEqual([
|
|
126
122
|
{
|
|
127
123
|
bucket: request.bucket,
|
|
@@ -163,9 +159,9 @@ bucket_definitions:
|
|
|
163
159
|
afterReplicaId: test_utils.rid('test1')
|
|
164
160
|
});
|
|
165
161
|
await writer.commit('2/1');
|
|
166
|
-
const
|
|
162
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
167
163
|
const request = bucketRequest(syncRules, 'global[]');
|
|
168
|
-
const batch = await test_utils.
|
|
164
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
169
165
|
const data = batch[0].chunkData.data.map((d) => {
|
|
170
166
|
return {
|
|
171
167
|
op: d.op,
|
|
@@ -175,9 +171,7 @@ bucket_definitions:
|
|
|
175
171
|
});
|
|
176
172
|
const c1 = 2871785649;
|
|
177
173
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
178
|
-
const checksums = [
|
|
179
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
180
|
-
];
|
|
174
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
181
175
|
expect(checksums).toEqual([
|
|
182
176
|
{
|
|
183
177
|
bucket: request.bucket,
|
|
@@ -215,9 +209,9 @@ bucket_definitions:
|
|
|
215
209
|
afterReplicaId: test_utils.rid('test1')
|
|
216
210
|
});
|
|
217
211
|
await writer.commit('1/1');
|
|
218
|
-
const
|
|
212
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
219
213
|
const request = bucketRequest(syncRules, 'global[]');
|
|
220
|
-
const batch = await test_utils.
|
|
214
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
221
215
|
const data = batch[0].chunkData.data.map((d) => {
|
|
222
216
|
return {
|
|
223
217
|
op: d.op,
|
|
@@ -227,9 +221,7 @@ bucket_definitions:
|
|
|
227
221
|
});
|
|
228
222
|
const c1 = 2871785649;
|
|
229
223
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
230
|
-
const checksums = [
|
|
231
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
232
|
-
];
|
|
224
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
233
225
|
expect(checksums).toEqual([
|
|
234
226
|
{
|
|
235
227
|
bucket: request.bucket,
|
|
@@ -289,9 +281,9 @@ bucket_definitions:
|
|
|
289
281
|
});
|
|
290
282
|
await writer.commit('2/1');
|
|
291
283
|
}
|
|
292
|
-
const
|
|
284
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
293
285
|
const request = bucketRequest(syncRules, 'global[]');
|
|
294
|
-
const batch = await test_utils.
|
|
286
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
295
287
|
expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
|
|
296
288
|
const data = batch[0].chunkData.data.map((d) => {
|
|
297
289
|
return {
|
|
@@ -347,8 +339,8 @@ bucket_definitions:
|
|
|
347
339
|
afterReplicaId: test_utils.rid('test2')
|
|
348
340
|
});
|
|
349
341
|
await writer.commit('1/1');
|
|
350
|
-
const
|
|
351
|
-
const batch = await test_utils.
|
|
342
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
343
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
|
|
352
344
|
const data = batch[0].chunkData.data.map((d) => {
|
|
353
345
|
return {
|
|
354
346
|
op: d.op,
|
|
@@ -398,9 +390,9 @@ bucket_definitions:
|
|
|
398
390
|
beforeReplicaId: test_utils.rid('test1')
|
|
399
391
|
});
|
|
400
392
|
await writer.flush();
|
|
401
|
-
const
|
|
393
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
402
394
|
const request = bucketRequest(syncRules, 'global[]');
|
|
403
|
-
const batch = await test_utils.
|
|
395
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
404
396
|
const data = batch[0].chunkData.data.map((d) => {
|
|
405
397
|
return {
|
|
406
398
|
op: d.op,
|
|
@@ -414,9 +406,7 @@ bucket_definitions:
|
|
|
414
406
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
415
407
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
416
408
|
]);
|
|
417
|
-
const checksums = [
|
|
418
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
419
|
-
];
|
|
409
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
420
410
|
expect(checksums).toEqual([
|
|
421
411
|
{
|
|
422
412
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -497,9 +487,9 @@ bucket_definitions:
|
|
|
497
487
|
beforeReplicaId: test_utils.rid('test1')
|
|
498
488
|
});
|
|
499
489
|
await writer.commit('2/1');
|
|
500
|
-
const
|
|
490
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
501
491
|
const request = bucketRequest(syncRules, 'global[]');
|
|
502
|
-
const batch = await test_utils.
|
|
492
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
503
493
|
const data = batch[0].chunkData.data.map((d) => {
|
|
504
494
|
return {
|
|
505
495
|
op: d.op,
|
|
@@ -515,9 +505,7 @@ bucket_definitions:
|
|
|
515
505
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
516
506
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
517
507
|
]);
|
|
518
|
-
const checksums = [
|
|
519
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
520
|
-
];
|
|
508
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
521
509
|
expect(checksums).toEqual([
|
|
522
510
|
{
|
|
523
511
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -639,7 +627,7 @@ bucket_definitions:
|
|
|
639
627
|
const result2 = await writer.flush();
|
|
640
628
|
const checkpoint2 = result2.flushed_op;
|
|
641
629
|
const request = bucketRequest(syncRules, 'global[]', checkpoint1);
|
|
642
|
-
const batch = await test_utils.
|
|
630
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request]));
|
|
643
631
|
const data = batch[0].chunkData.data.map((d) => {
|
|
644
632
|
return {
|
|
645
633
|
op: d.op,
|
|
@@ -725,7 +713,7 @@ bucket_definitions:
|
|
|
725
713
|
const result3 = await writer.flush();
|
|
726
714
|
const checkpoint3 = result3.flushed_op;
|
|
727
715
|
const request = bucketRequest(syncRules, 'global[]');
|
|
728
|
-
const batch = await test_utils.
|
|
716
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
|
|
729
717
|
const data = batch[0].chunkData.data.map((d) => {
|
|
730
718
|
return {
|
|
731
719
|
op: d.op,
|
|
@@ -821,7 +809,7 @@ bucket_definitions:
|
|
|
821
809
|
const result3 = await writer.flush();
|
|
822
810
|
const checkpoint3 = result3.flushed_op;
|
|
823
811
|
const request = bucketRequest(syncRules, 'global[]');
|
|
824
|
-
const batch = await test_utils.
|
|
812
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
|
|
825
813
|
const data = batch[0].chunkData.data.map((d) => {
|
|
826
814
|
return {
|
|
827
815
|
op: d.op,
|
|
@@ -905,12 +893,12 @@ bucket_definitions:
|
|
|
905
893
|
afterReplicaId: test_utils.rid('test3')
|
|
906
894
|
});
|
|
907
895
|
await writer.commit('1/1');
|
|
908
|
-
const
|
|
896
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
909
897
|
const options = {
|
|
910
898
|
chunkLimitBytes: 16 * 1024 * 1024
|
|
911
899
|
};
|
|
912
900
|
const request = bucketRequest(syncRules, 'global[]');
|
|
913
|
-
const batch1 = await test_utils.
|
|
901
|
+
const batch1 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
|
|
914
902
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
915
903
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
916
904
|
{ op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
|
|
@@ -920,7 +908,7 @@ bucket_definitions:
|
|
|
920
908
|
has_more: true,
|
|
921
909
|
next_after: '2'
|
|
922
910
|
});
|
|
923
|
-
const batch2 = await test_utils.
|
|
911
|
+
const batch2 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
|
|
924
912
|
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
925
913
|
{ op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
|
|
926
914
|
{ op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
|
|
@@ -930,12 +918,13 @@ bucket_definitions:
|
|
|
930
918
|
has_more: false,
|
|
931
919
|
next_after: '4'
|
|
932
920
|
});
|
|
933
|
-
const batch3 = await test_utils.
|
|
921
|
+
const batch3 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
|
|
934
922
|
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
935
923
|
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
936
924
|
});
|
|
937
925
|
test('long batch', async () => {
|
|
938
|
-
// Test syncing a batch of data that is limited by count.
|
|
926
|
+
// Test syncing a batch of data that is limited by count. Compressed storage limits persisted
|
|
927
|
+
// documents and finishes yielding all operations from a document once it has been hydrated.
|
|
939
928
|
await using factory = await generateStorageFactory();
|
|
940
929
|
const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
|
|
941
930
|
bucket_definitions:
|
|
@@ -961,40 +950,55 @@ bucket_definitions:
|
|
|
961
950
|
});
|
|
962
951
|
}
|
|
963
952
|
await writer.commit('1/1');
|
|
964
|
-
const
|
|
953
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
965
954
|
const request = bucketRequest(syncRules, 'global[]');
|
|
966
|
-
const batch1 = await test_utils.
|
|
967
|
-
|
|
955
|
+
const batch1 = await test_utils.getSingleBatchItem(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
|
|
956
|
+
const allOperations = [
|
|
968
957
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
969
958
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
970
959
|
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
971
|
-
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
|
|
972
|
-
]);
|
|
973
|
-
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
974
|
-
after: '0',
|
|
975
|
-
has_more: true,
|
|
976
|
-
next_after: '4'
|
|
977
|
-
});
|
|
978
|
-
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
979
|
-
limit: 4
|
|
980
|
-
}));
|
|
981
|
-
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
960
|
+
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
|
|
982
961
|
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
983
962
|
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
|
|
984
|
-
]
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
963
|
+
];
|
|
964
|
+
if (config.compressedBucketStorage) {
|
|
965
|
+
expect(test_utils.getBatchData(batch1)).toEqual(allOperations);
|
|
966
|
+
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
967
|
+
after: '0',
|
|
968
|
+
has_more: false,
|
|
969
|
+
next_after: '6'
|
|
970
|
+
});
|
|
971
|
+
const batch2 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
972
|
+
limit: 4
|
|
973
|
+
}));
|
|
974
|
+
expect(test_utils.getBatchData(batch2)).toEqual([]);
|
|
975
|
+
expect(test_utils.getBatchMeta(batch2)).toEqual(null);
|
|
976
|
+
}
|
|
977
|
+
else {
|
|
978
|
+
expect(test_utils.getBatchData(batch1)).toEqual(allOperations.slice(0, 4));
|
|
979
|
+
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
980
|
+
after: '0',
|
|
981
|
+
has_more: true,
|
|
982
|
+
next_after: '4'
|
|
983
|
+
});
|
|
984
|
+
const batch2 = await test_utils.getSingleBatchItem(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
985
|
+
limit: 4
|
|
986
|
+
}));
|
|
987
|
+
expect(test_utils.getBatchData(batch2)).toEqual(allOperations.slice(4));
|
|
988
|
+
expect(test_utils.getBatchMeta(batch2)).toEqual({
|
|
989
|
+
after: '4',
|
|
990
|
+
has_more: false,
|
|
991
|
+
next_after: '6'
|
|
992
|
+
});
|
|
993
|
+
const batch3 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
|
|
994
|
+
limit: 4
|
|
995
|
+
}));
|
|
996
|
+
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
997
|
+
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
998
|
+
}
|
|
995
999
|
});
|
|
996
1000
|
describe('batch has_more', () => {
|
|
997
|
-
const setup = async (options) => {
|
|
1001
|
+
const setup = async (options, commitAfter) => {
|
|
998
1002
|
await using factory = await generateStorageFactory();
|
|
999
1003
|
const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
|
|
1000
1004
|
bucket_definitions:
|
|
@@ -1020,28 +1024,39 @@ bucket_definitions:
|
|
|
1020
1024
|
},
|
|
1021
1025
|
afterReplicaId: `test${i}`
|
|
1022
1026
|
});
|
|
1027
|
+
if (i == commitAfter) {
|
|
1028
|
+
await writer.commit('1/1');
|
|
1029
|
+
}
|
|
1023
1030
|
}
|
|
1024
|
-
await writer.commit('1/1');
|
|
1025
|
-
const
|
|
1031
|
+
await writer.commit(commitAfter == null ? '1/1' : '1/2');
|
|
1032
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1026
1033
|
const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
|
|
1027
1034
|
const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
|
|
1028
|
-
const batch = await test_utils.
|
|
1035
|
+
const batch = await new test_utils.StorageDataHelpers(bucketStorage, syncRules).getAllBucketData([global1Request, global2Request], checkpoint, options);
|
|
1029
1036
|
return { batch, global1Request, global2Request };
|
|
1030
1037
|
};
|
|
1031
1038
|
test('batch has_more (1)', async () => {
|
|
1032
|
-
|
|
1033
|
-
|
|
1039
|
+
// The commit boundary creates two compressed documents for global2. The limit selects the global1
|
|
1040
|
+
// document and the first global2 document, leaving the second global2 document for the next batch.
|
|
1041
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 2 }, 5);
|
|
1042
|
+
expect(batch.length).toBeGreaterThanOrEqual(2);
|
|
1034
1043
|
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1035
1044
|
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1036
1045
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1037
1046
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1038
1047
|
]);
|
|
1039
|
-
|
|
1048
|
+
const global2Operations = [
|
|
1040
1049
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
1041
1050
|
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
1042
1051
|
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
|
|
1043
|
-
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }
|
|
1044
|
-
|
|
1052
|
+
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
1053
|
+
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
|
|
1054
|
+
{ op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
|
|
1055
|
+
{ op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
|
|
1056
|
+
{ op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
|
|
1057
|
+
{ op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
|
|
1058
|
+
];
|
|
1059
|
+
expect(test_utils.getBatchData(batch[1])).toEqual(config.compressedBucketStorage ? global2Operations.slice(0, 4) : global2Operations.slice(0, 1));
|
|
1045
1060
|
expect(test_utils.getBatchMeta(batch[0])).toEqual({
|
|
1046
1061
|
after: '0',
|
|
1047
1062
|
has_more: false,
|
|
@@ -1050,7 +1065,7 @@ bucket_definitions:
|
|
|
1050
1065
|
expect(test_utils.getBatchMeta(batch[1])).toEqual({
|
|
1051
1066
|
after: '0',
|
|
1052
1067
|
has_more: true,
|
|
1053
|
-
next_after: '5'
|
|
1068
|
+
next_after: config.compressedBucketStorage ? '5' : '2'
|
|
1054
1069
|
});
|
|
1055
1070
|
});
|
|
1056
1071
|
test('batch has_more (2)', async () => {
|
|
@@ -1087,7 +1102,8 @@ bucket_definitions:
|
|
|
1087
1102
|
// 50 bytes is more than 1 row, less than 2 rows
|
|
1088
1103
|
const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
|
|
1089
1104
|
if (config.compressedBucketStorage) {
|
|
1090
|
-
// In v3+, ops in the same bucket share a document
|
|
1105
|
+
// In v3+, ops in the same bucket share a document. Once hydrated, the entire document is yielded even
|
|
1106
|
+
// when it exceeds the byte limit; the byte limit controls whether another document is fetched.
|
|
1091
1107
|
expect(batch.length).toEqual(2);
|
|
1092
1108
|
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1093
1109
|
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
@@ -1096,7 +1112,14 @@ bucket_definitions:
|
|
|
1096
1112
|
]);
|
|
1097
1113
|
expect(test_utils.getBatchData(batch[1])).toEqual([
|
|
1098
1114
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
1099
|
-
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
|
|
1115
|
+
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
1116
|
+
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
|
|
1117
|
+
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
1118
|
+
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
|
|
1119
|
+
{ op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
|
|
1120
|
+
{ op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
|
|
1121
|
+
{ op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
|
|
1122
|
+
{ op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
|
|
1100
1123
|
]);
|
|
1101
1124
|
expect(test_utils.getBatchMeta(batch[0])).toEqual({
|
|
1102
1125
|
after: '0',
|
|
@@ -1105,12 +1128,12 @@ bucket_definitions:
|
|
|
1105
1128
|
});
|
|
1106
1129
|
expect(test_utils.getBatchMeta(batch[1])).toEqual({
|
|
1107
1130
|
after: '0',
|
|
1108
|
-
has_more:
|
|
1109
|
-
next_after: '
|
|
1131
|
+
has_more: false,
|
|
1132
|
+
next_after: '10'
|
|
1110
1133
|
});
|
|
1111
1134
|
}
|
|
1112
1135
|
else {
|
|
1113
|
-
expect(batch.length).
|
|
1136
|
+
expect(batch.length).toBeGreaterThanOrEqual(3);
|
|
1114
1137
|
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1115
1138
|
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1116
1139
|
expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
|
|
@@ -1225,21 +1248,19 @@ bucket_definitions:
|
|
|
1225
1248
|
afterReplicaId: test_utils.rid('test1')
|
|
1226
1249
|
});
|
|
1227
1250
|
await writer.commit('1/1');
|
|
1228
|
-
const
|
|
1251
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1229
1252
|
const request = bucketRequest(syncRules, 'global[]');
|
|
1230
|
-
const checksums = [
|
|
1231
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
1232
|
-
];
|
|
1253
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
1233
1254
|
expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1234
1255
|
const checksums2 = [
|
|
1235
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint + 1n), [request])).values()
|
|
1256
|
+
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint.checkpoint + 1n), [request])).values()
|
|
1236
1257
|
];
|
|
1237
1258
|
expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1238
1259
|
});
|
|
1239
1260
|
testChecksumBatching(config);
|
|
1240
1261
|
test('empty checkpoints (1)', async () => {
|
|
1241
1262
|
await using factory = await generateStorageFactory();
|
|
1242
|
-
const { stream: replicationStream
|
|
1263
|
+
const { stream: replicationStream } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
|
|
1243
1264
|
bucket_definitions:
|
|
1244
1265
|
global:
|
|
1245
1266
|
data:
|
|
@@ -1265,7 +1286,7 @@ bucket_definitions:
|
|
|
1265
1286
|
});
|
|
1266
1287
|
test('empty checkpoints (2)', async () => {
|
|
1267
1288
|
await using factory = await generateStorageFactory();
|
|
1268
|
-
const { stream: replicationStream
|
|
1289
|
+
const { stream: replicationStream } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
|
|
1269
1290
|
bucket_definitions:
|
|
1270
1291
|
global:
|
|
1271
1292
|
data:
|
|
@@ -1385,7 +1406,7 @@ bucket_definitions:
|
|
|
1385
1406
|
await streamingWriter.keepalive('3/1');
|
|
1386
1407
|
const cp = await bucketStorage.getCheckpoint();
|
|
1387
1408
|
expect(cp.lsn).toEqual('3/1');
|
|
1388
|
-
const data = await test_utils.
|
|
1409
|
+
const data = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(cp, [bucketRequest(syncRules, 'global[]')]));
|
|
1389
1410
|
expect(data).toEqual([]);
|
|
1390
1411
|
});
|
|
1391
1412
|
}
|
|
@@ -1427,14 +1448,12 @@ bucket_definitions:
|
|
|
1427
1448
|
}
|
|
1428
1449
|
}
|
|
1429
1450
|
await writer.commit('1/1');
|
|
1430
|
-
const
|
|
1451
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1431
1452
|
bucketStorage.clearChecksumCache();
|
|
1432
1453
|
const users = ['u1', 'u2', 'u3', 'u4'];
|
|
1433
1454
|
const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
|
|
1434
1455
|
const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
|
|
1435
|
-
const checksums = [
|
|
1436
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), bucketRequests)).values()
|
|
1437
|
-
];
|
|
1456
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
|
|
1438
1457
|
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1439
1458
|
const expected = bucketRequests.map((request, index) => ({
|
|
1440
1459
|
bucket: request.bucket,
|