@powersync/service-core-tests 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/dist/test-utils/AbstractStreamTestContext.d.ts +1 -1
- package/dist/test-utils/AbstractStreamTestContext.js +2 -3
- package/dist/test-utils/AbstractStreamTestContext.js.map +1 -1
- package/dist/test-utils/StorageDataHelpers.d.ts +3 -2
- package/dist/test-utils/StorageDataHelpers.js +40 -2
- package/dist/test-utils/StorageDataHelpers.js.map +1 -1
- package/dist/test-utils/general-utils.d.ts +3 -1
- package/dist/test-utils/general-utils.js +13 -1
- package/dist/test-utils/general-utils.js.map +1 -1
- package/dist/test-utils/stream_utils.d.ts +0 -1
- package/dist/test-utils/stream_utils.js +0 -10
- package/dist/test-utils/stream_utils.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +7 -7
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +96 -3
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +113 -94
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +259 -75
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/package.json +5 -5
- package/src/test-utils/AbstractStreamTestContext.ts +2 -3
- package/src/test-utils/StorageDataHelpers.ts +45 -2
- package/src/test-utils/general-utils.ts +27 -3
- package/src/test-utils/stream_utils.ts +0 -11
- package/src/tests/register-compacting-tests.ts +7 -7
- package/src/tests/register-data-storage-checkpoint-tests.ts +122 -3
- package/src/tests/register-data-storage-data-tests.ts +129 -132
- package/src/tests/register-sync-tests.ts +280 -77
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -72,12 +72,10 @@ bucket_definitions:
|
|
|
72
72
|
});
|
|
73
73
|
await writer.commit('1/1');
|
|
74
74
|
|
|
75
|
-
const
|
|
75
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
76
76
|
|
|
77
77
|
const request = bucketRequest(syncRules, 'global[]');
|
|
78
|
-
const batch = await test_utils.
|
|
79
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
80
|
-
);
|
|
78
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
81
79
|
const data = batch[0].chunkData.data.map((d) => {
|
|
82
80
|
return {
|
|
83
81
|
op: d.op,
|
|
@@ -94,9 +92,7 @@ bucket_definitions:
|
|
|
94
92
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
95
93
|
]);
|
|
96
94
|
|
|
97
|
-
const checksums = [
|
|
98
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
99
|
-
];
|
|
95
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
100
96
|
expect(checksums).toEqual([
|
|
101
97
|
{
|
|
102
98
|
bucket: request.bucket,
|
|
@@ -144,12 +140,10 @@ bucket_definitions:
|
|
|
144
140
|
});
|
|
145
141
|
await writer.commit('2/1');
|
|
146
142
|
|
|
147
|
-
const
|
|
143
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
148
144
|
|
|
149
145
|
const request = bucketRequest(syncRules, 'global[]');
|
|
150
|
-
const batch = await test_utils.
|
|
151
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
152
|
-
);
|
|
146
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
153
147
|
const data = batch[0].chunkData.data.map((d) => {
|
|
154
148
|
return {
|
|
155
149
|
op: d.op,
|
|
@@ -162,9 +156,7 @@ bucket_definitions:
|
|
|
162
156
|
|
|
163
157
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
164
158
|
|
|
165
|
-
const checksums = [
|
|
166
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
167
|
-
];
|
|
159
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
168
160
|
expect(checksums).toEqual([
|
|
169
161
|
{
|
|
170
162
|
bucket: request.bucket,
|
|
@@ -217,12 +209,10 @@ bucket_definitions:
|
|
|
217
209
|
});
|
|
218
210
|
await writer.commit('2/1');
|
|
219
211
|
|
|
220
|
-
const
|
|
212
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
221
213
|
|
|
222
214
|
const request = bucketRequest(syncRules, 'global[]');
|
|
223
|
-
const batch = await test_utils.
|
|
224
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
225
|
-
);
|
|
215
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
226
216
|
const data = batch[0].chunkData.data.map((d) => {
|
|
227
217
|
return {
|
|
228
218
|
op: d.op,
|
|
@@ -235,9 +225,7 @@ bucket_definitions:
|
|
|
235
225
|
|
|
236
226
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
237
227
|
|
|
238
|
-
const checksums = [
|
|
239
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
240
|
-
];
|
|
228
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
241
229
|
expect(checksums).toEqual([
|
|
242
230
|
{
|
|
243
231
|
bucket: request.bucket,
|
|
@@ -284,12 +272,10 @@ bucket_definitions:
|
|
|
284
272
|
});
|
|
285
273
|
await writer.commit('1/1');
|
|
286
274
|
|
|
287
|
-
const
|
|
275
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
288
276
|
|
|
289
277
|
const request = bucketRequest(syncRules, 'global[]');
|
|
290
|
-
const batch = await test_utils.
|
|
291
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
292
|
-
);
|
|
278
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
293
279
|
const data = batch[0].chunkData.data.map((d) => {
|
|
294
280
|
return {
|
|
295
281
|
op: d.op,
|
|
@@ -302,9 +288,7 @@ bucket_definitions:
|
|
|
302
288
|
|
|
303
289
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
304
290
|
|
|
305
|
-
const checksums = [
|
|
306
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
307
|
-
];
|
|
291
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
308
292
|
expect(checksums).toEqual([
|
|
309
293
|
{
|
|
310
294
|
bucket: request.bucket,
|
|
@@ -375,12 +359,10 @@ bucket_definitions:
|
|
|
375
359
|
await writer.commit('2/1');
|
|
376
360
|
}
|
|
377
361
|
|
|
378
|
-
const
|
|
362
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
379
363
|
|
|
380
364
|
const request = bucketRequest(syncRules, 'global[]');
|
|
381
|
-
const batch = await test_utils.
|
|
382
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
383
|
-
);
|
|
365
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
384
366
|
|
|
385
367
|
expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
|
|
386
368
|
|
|
@@ -448,9 +430,9 @@ bucket_definitions:
|
|
|
448
430
|
});
|
|
449
431
|
|
|
450
432
|
await writer.commit('1/1');
|
|
451
|
-
const
|
|
452
|
-
const batch = await test_utils.
|
|
453
|
-
bucketStorage.getBucketDataBatch(
|
|
433
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
434
|
+
const batch = await test_utils.getBatchArray(
|
|
435
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')])
|
|
454
436
|
);
|
|
455
437
|
const data = batch[0].chunkData.data.map((d) => {
|
|
456
438
|
return {
|
|
@@ -514,12 +496,10 @@ bucket_definitions:
|
|
|
514
496
|
});
|
|
515
497
|
await writer.flush();
|
|
516
498
|
|
|
517
|
-
const
|
|
499
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
518
500
|
|
|
519
501
|
const request = bucketRequest(syncRules, 'global[]');
|
|
520
|
-
const batch = await test_utils.
|
|
521
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
522
|
-
);
|
|
502
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
523
503
|
const data = batch[0].chunkData.data.map((d) => {
|
|
524
504
|
return {
|
|
525
505
|
op: d.op,
|
|
@@ -536,9 +516,7 @@ bucket_definitions:
|
|
|
536
516
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
537
517
|
]);
|
|
538
518
|
|
|
539
|
-
const checksums = [
|
|
540
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
541
|
-
];
|
|
519
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
542
520
|
expect(checksums).toEqual([
|
|
543
521
|
{
|
|
544
522
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -638,12 +616,10 @@ bucket_definitions:
|
|
|
638
616
|
|
|
639
617
|
await writer.commit('2/1');
|
|
640
618
|
|
|
641
|
-
const
|
|
619
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
642
620
|
|
|
643
621
|
const request = bucketRequest(syncRules, 'global[]');
|
|
644
|
-
const batch = await test_utils.
|
|
645
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
|
|
646
|
-
);
|
|
622
|
+
const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
647
623
|
|
|
648
624
|
const data = batch[0].chunkData.data.map((d) => {
|
|
649
625
|
return {
|
|
@@ -663,9 +639,7 @@ bucket_definitions:
|
|
|
663
639
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
664
640
|
]);
|
|
665
641
|
|
|
666
|
-
const checksums = [
|
|
667
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
668
|
-
];
|
|
642
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
669
643
|
expect(checksums).toEqual([
|
|
670
644
|
{
|
|
671
645
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -808,7 +782,7 @@ bucket_definitions:
|
|
|
808
782
|
const checkpoint2 = result2!.flushed_op;
|
|
809
783
|
|
|
810
784
|
const request = bucketRequest(syncRules, 'global[]', checkpoint1);
|
|
811
|
-
const batch = await test_utils.
|
|
785
|
+
const batch = await test_utils.getBatchArray(
|
|
812
786
|
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request])
|
|
813
787
|
);
|
|
814
788
|
|
|
@@ -914,7 +888,7 @@ bucket_definitions:
|
|
|
914
888
|
const checkpoint3 = result3!.flushed_op;
|
|
915
889
|
|
|
916
890
|
const request = bucketRequest(syncRules, 'global[]');
|
|
917
|
-
const batch = await test_utils.
|
|
891
|
+
const batch = await test_utils.getBatchArray(
|
|
918
892
|
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }])
|
|
919
893
|
);
|
|
920
894
|
const data = batch[0].chunkData.data.map((d) => {
|
|
@@ -1028,7 +1002,7 @@ bucket_definitions:
|
|
|
1028
1002
|
const checkpoint3 = result3!.flushed_op;
|
|
1029
1003
|
|
|
1030
1004
|
const request = bucketRequest(syncRules, 'global[]');
|
|
1031
|
-
const batch = await test_utils.
|
|
1005
|
+
const batch = await test_utils.getBatchArray(
|
|
1032
1006
|
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }])
|
|
1033
1007
|
);
|
|
1034
1008
|
const data = batch[0].chunkData.data.map((d) => {
|
|
@@ -1129,16 +1103,14 @@ bucket_definitions:
|
|
|
1129
1103
|
|
|
1130
1104
|
await writer.commit('1/1');
|
|
1131
1105
|
|
|
1132
|
-
const
|
|
1106
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1133
1107
|
|
|
1134
1108
|
const options: storage.BucketDataBatchOptions = {
|
|
1135
1109
|
chunkLimitBytes: 16 * 1024 * 1024
|
|
1136
1110
|
};
|
|
1137
1111
|
|
|
1138
1112
|
const request = bucketRequest(syncRules, 'global[]');
|
|
1139
|
-
const batch1 = await test_utils.
|
|
1140
|
-
bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request], options)
|
|
1141
|
-
);
|
|
1113
|
+
const batch1 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
|
|
1142
1114
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
1143
1115
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
1144
1116
|
{ op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
|
|
@@ -1149,9 +1121,9 @@ bucket_definitions:
|
|
|
1149
1121
|
next_after: '2'
|
|
1150
1122
|
});
|
|
1151
1123
|
|
|
1152
|
-
const batch2 = await test_utils.
|
|
1124
|
+
const batch2 = await test_utils.getBatchArray(
|
|
1153
1125
|
bucketStorage.getBucketDataBatch(
|
|
1154
|
-
|
|
1126
|
+
checkpoint,
|
|
1155
1127
|
[{ ...request, start: BigInt(batch1[0].chunkData.next_after) }],
|
|
1156
1128
|
options
|
|
1157
1129
|
)
|
|
@@ -1166,9 +1138,9 @@ bucket_definitions:
|
|
|
1166
1138
|
next_after: '4'
|
|
1167
1139
|
});
|
|
1168
1140
|
|
|
1169
|
-
const batch3 = await test_utils.
|
|
1141
|
+
const batch3 = await test_utils.getBatchArray(
|
|
1170
1142
|
bucketStorage.getBucketDataBatch(
|
|
1171
|
-
|
|
1143
|
+
checkpoint,
|
|
1172
1144
|
[{ ...request, start: BigInt(batch2[0].chunkData.next_after) }],
|
|
1173
1145
|
options
|
|
1174
1146
|
)
|
|
@@ -1178,7 +1150,8 @@ bucket_definitions:
|
|
|
1178
1150
|
});
|
|
1179
1151
|
|
|
1180
1152
|
test('long batch', async () => {
|
|
1181
|
-
// Test syncing a batch of data that is limited by count.
|
|
1153
|
+
// Test syncing a batch of data that is limited by count. Compressed storage limits persisted
|
|
1154
|
+
// documents and finishes yielding all operations from a document once it has been hydrated.
|
|
1182
1155
|
await using factory = await generateStorageFactory();
|
|
1183
1156
|
const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
|
|
1184
1157
|
factory,
|
|
@@ -1213,62 +1186,71 @@ bucket_definitions:
|
|
|
1213
1186
|
|
|
1214
1187
|
await writer.commit('1/1');
|
|
1215
1188
|
|
|
1216
|
-
const
|
|
1189
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1217
1190
|
|
|
1218
1191
|
const request = bucketRequest(syncRules, 'global[]');
|
|
1219
|
-
const batch1 = await test_utils.
|
|
1220
|
-
bucketStorage.getBucketDataBatch(
|
|
1192
|
+
const batch1 = await test_utils.getSingleBatchItem(
|
|
1193
|
+
bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 })
|
|
1221
1194
|
);
|
|
1222
1195
|
|
|
1223
|
-
|
|
1196
|
+
const allOperations = [
|
|
1224
1197
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
1225
1198
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
1226
1199
|
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
1227
|
-
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
|
|
1228
|
-
|
|
1200
|
+
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
|
|
1201
|
+
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
1202
|
+
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
|
|
1203
|
+
];
|
|
1229
1204
|
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1205
|
+
if (config.compressedBucketStorage) {
|
|
1206
|
+
expect(test_utils.getBatchData(batch1)).toEqual(allOperations);
|
|
1207
|
+
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
1208
|
+
after: '0',
|
|
1209
|
+
has_more: false,
|
|
1210
|
+
next_after: '6'
|
|
1211
|
+
});
|
|
1235
1212
|
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
test_utils.testCheckpoint(checkpoint),
|
|
1239
|
-
[{ ...request, start: BigInt(batch1.chunkData.next_after) }],
|
|
1240
|
-
{
|
|
1213
|
+
const batch2 = await test_utils.getBatchArray(
|
|
1214
|
+
bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
1241
1215
|
limit: 4
|
|
1242
|
-
}
|
|
1243
|
-
)
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
]);
|
|
1216
|
+
})
|
|
1217
|
+
);
|
|
1218
|
+
expect(test_utils.getBatchData(batch2)).toEqual([]);
|
|
1219
|
+
expect(test_utils.getBatchMeta(batch2)).toEqual(null);
|
|
1220
|
+
} else {
|
|
1221
|
+
expect(test_utils.getBatchData(batch1)).toEqual(allOperations.slice(0, 4));
|
|
1249
1222
|
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1223
|
+
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
1224
|
+
after: '0',
|
|
1225
|
+
has_more: true,
|
|
1226
|
+
next_after: '4'
|
|
1227
|
+
});
|
|
1255
1228
|
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
test_utils.testCheckpoint(checkpoint),
|
|
1259
|
-
[{ ...request, start: BigInt(batch2.chunkData.next_after) }],
|
|
1260
|
-
{
|
|
1229
|
+
const batch2 = await test_utils.getSingleBatchItem(
|
|
1230
|
+
bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
1261
1231
|
limit: 4
|
|
1262
|
-
}
|
|
1263
|
-
)
|
|
1264
|
-
|
|
1265
|
-
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
1232
|
+
})
|
|
1233
|
+
);
|
|
1234
|
+
expect(test_utils.getBatchData(batch2)).toEqual(allOperations.slice(4));
|
|
1266
1235
|
|
|
1267
|
-
|
|
1236
|
+
expect(test_utils.getBatchMeta(batch2)).toEqual({
|
|
1237
|
+
after: '4',
|
|
1238
|
+
has_more: false,
|
|
1239
|
+
next_after: '6'
|
|
1240
|
+
});
|
|
1241
|
+
|
|
1242
|
+
const batch3 = await test_utils.getBatchArray(
|
|
1243
|
+
bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
|
|
1244
|
+
limit: 4
|
|
1245
|
+
})
|
|
1246
|
+
);
|
|
1247
|
+
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
1248
|
+
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
1249
|
+
}
|
|
1268
1250
|
});
|
|
1269
1251
|
|
|
1270
1252
|
describe('batch has_more', () => {
|
|
1271
|
-
const setup = async (options: BucketDataBatchOptions) => {
|
|
1253
|
+
const setup = async (options: BucketDataBatchOptions, commitAfter?: number) => {
|
|
1272
1254
|
await using factory = await generateStorageFactory();
|
|
1273
1255
|
const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
|
|
1274
1256
|
factory,
|
|
@@ -1301,27 +1283,30 @@ bucket_definitions:
|
|
|
1301
1283
|
},
|
|
1302
1284
|
afterReplicaId: `test${i}`
|
|
1303
1285
|
});
|
|
1286
|
+
if (i == commitAfter) {
|
|
1287
|
+
await writer.commit('1/1');
|
|
1288
|
+
}
|
|
1304
1289
|
}
|
|
1305
1290
|
|
|
1306
|
-
await writer.commit('1/1');
|
|
1291
|
+
await writer.commit(commitAfter == null ? '1/1' : '1/2');
|
|
1307
1292
|
|
|
1308
|
-
const
|
|
1293
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1309
1294
|
const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
|
|
1310
1295
|
const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
|
|
1311
|
-
const batch = await test_utils.
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
options
|
|
1316
|
-
)
|
|
1296
|
+
const batch = await new test_utils.StorageDataHelpers(bucketStorage, syncRules).getAllBucketData(
|
|
1297
|
+
[global1Request, global2Request],
|
|
1298
|
+
checkpoint,
|
|
1299
|
+
options
|
|
1317
1300
|
);
|
|
1318
1301
|
|
|
1319
1302
|
return { batch, global1Request, global2Request };
|
|
1320
1303
|
};
|
|
1321
1304
|
|
|
1322
1305
|
test('batch has_more (1)', async () => {
|
|
1323
|
-
|
|
1324
|
-
|
|
1306
|
+
// The commit boundary creates two compressed documents for global2. The limit selects the global1
|
|
1307
|
+
// document and the first global2 document, leaving the second global2 document for the next batch.
|
|
1308
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 2 }, 5);
|
|
1309
|
+
expect(batch.length).toBeGreaterThanOrEqual(2);
|
|
1325
1310
|
|
|
1326
1311
|
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1327
1312
|
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
@@ -1330,12 +1315,20 @@ bucket_definitions:
|
|
|
1330
1315
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1331
1316
|
]);
|
|
1332
1317
|
|
|
1333
|
-
|
|
1318
|
+
const global2Operations = [
|
|
1334
1319
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
1335
1320
|
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
1336
1321
|
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
|
|
1337
|
-
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }
|
|
1338
|
-
|
|
1322
|
+
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
1323
|
+
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
|
|
1324
|
+
{ op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
|
|
1325
|
+
{ op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
|
|
1326
|
+
{ op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
|
|
1327
|
+
{ op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
|
|
1328
|
+
];
|
|
1329
|
+
expect(test_utils.getBatchData(batch[1])).toEqual(
|
|
1330
|
+
config.compressedBucketStorage ? global2Operations.slice(0, 4) : global2Operations.slice(0, 1)
|
|
1331
|
+
);
|
|
1339
1332
|
|
|
1340
1333
|
expect(test_utils.getBatchMeta(batch[0])).toEqual({
|
|
1341
1334
|
after: '0',
|
|
@@ -1346,7 +1339,7 @@ bucket_definitions:
|
|
|
1346
1339
|
expect(test_utils.getBatchMeta(batch[1])).toEqual({
|
|
1347
1340
|
after: '0',
|
|
1348
1341
|
has_more: true,
|
|
1349
|
-
next_after: '5'
|
|
1342
|
+
next_after: config.compressedBucketStorage ? '5' : '2'
|
|
1350
1343
|
});
|
|
1351
1344
|
});
|
|
1352
1345
|
|
|
@@ -1391,7 +1384,8 @@ bucket_definitions:
|
|
|
1391
1384
|
const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
|
|
1392
1385
|
|
|
1393
1386
|
if (config.compressedBucketStorage) {
|
|
1394
|
-
// In v3+, ops in the same bucket share a document
|
|
1387
|
+
// In v3+, ops in the same bucket share a document. Once hydrated, the entire document is yielded even
|
|
1388
|
+
// when it exceeds the byte limit; the byte limit controls whether another document is fetched.
|
|
1395
1389
|
expect(batch.length).toEqual(2);
|
|
1396
1390
|
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1397
1391
|
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
@@ -1402,7 +1396,14 @@ bucket_definitions:
|
|
|
1402
1396
|
|
|
1403
1397
|
expect(test_utils.getBatchData(batch[1])).toEqual([
|
|
1404
1398
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
1405
|
-
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
|
|
1399
|
+
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
1400
|
+
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
|
|
1401
|
+
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
1402
|
+
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
|
|
1403
|
+
{ op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
|
|
1404
|
+
{ op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
|
|
1405
|
+
{ op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
|
|
1406
|
+
{ op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
|
|
1406
1407
|
]);
|
|
1407
1408
|
|
|
1408
1409
|
expect(test_utils.getBatchMeta(batch[0])).toEqual({
|
|
@@ -1413,11 +1414,11 @@ bucket_definitions:
|
|
|
1413
1414
|
|
|
1414
1415
|
expect(test_utils.getBatchMeta(batch[1])).toEqual({
|
|
1415
1416
|
after: '0',
|
|
1416
|
-
has_more:
|
|
1417
|
-
next_after: '
|
|
1417
|
+
has_more: false,
|
|
1418
|
+
next_after: '10'
|
|
1418
1419
|
});
|
|
1419
1420
|
} else {
|
|
1420
|
-
expect(batch.length).
|
|
1421
|
+
expect(batch.length).toBeGreaterThanOrEqual(3);
|
|
1421
1422
|
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1422
1423
|
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1423
1424
|
expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
|
|
@@ -1560,15 +1561,13 @@ bucket_definitions:
|
|
|
1560
1561
|
afterReplicaId: test_utils.rid('test1')
|
|
1561
1562
|
});
|
|
1562
1563
|
await writer.commit('1/1');
|
|
1563
|
-
const
|
|
1564
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1564
1565
|
|
|
1565
1566
|
const request = bucketRequest(syncRules, 'global[]');
|
|
1566
|
-
const checksums = [
|
|
1567
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
|
|
1568
|
-
];
|
|
1567
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
1569
1568
|
expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1570
1569
|
const checksums2 = [
|
|
1571
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint + 1n), [request])).values()
|
|
1570
|
+
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint.checkpoint + 1n), [request])).values()
|
|
1572
1571
|
];
|
|
1573
1572
|
expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1574
1573
|
});
|
|
@@ -1577,7 +1576,7 @@ bucket_definitions:
|
|
|
1577
1576
|
|
|
1578
1577
|
test('empty checkpoints (1)', async () => {
|
|
1579
1578
|
await using factory = await generateStorageFactory();
|
|
1580
|
-
const { stream: replicationStream
|
|
1579
|
+
const { stream: replicationStream } = await test_utils.deploySyncRules(
|
|
1581
1580
|
factory,
|
|
1582
1581
|
updateSyncRulesFromYaml(
|
|
1583
1582
|
`
|
|
@@ -1614,7 +1613,7 @@ bucket_definitions:
|
|
|
1614
1613
|
|
|
1615
1614
|
test('empty checkpoints (2)', async () => {
|
|
1616
1615
|
await using factory = await generateStorageFactory();
|
|
1617
|
-
const { stream: replicationStream
|
|
1616
|
+
const { stream: replicationStream } = await test_utils.deploySyncRules(
|
|
1618
1617
|
factory,
|
|
1619
1618
|
updateSyncRulesFromYaml(
|
|
1620
1619
|
`
|
|
@@ -1771,8 +1770,8 @@ bucket_definitions:
|
|
|
1771
1770
|
|
|
1772
1771
|
const cp = await bucketStorage.getCheckpoint();
|
|
1773
1772
|
expect(cp.lsn).toEqual('3/1');
|
|
1774
|
-
const data = await test_utils.
|
|
1775
|
-
bucketStorage.getBucketDataBatch(
|
|
1773
|
+
const data = await test_utils.getBatchArray(
|
|
1774
|
+
bucketStorage.getBucketDataBatch(cp, [bucketRequest(syncRules, 'global[]')])
|
|
1776
1775
|
);
|
|
1777
1776
|
|
|
1778
1777
|
expect(data).toEqual([]);
|
|
@@ -1823,15 +1822,13 @@ bucket_definitions:
|
|
|
1823
1822
|
}
|
|
1824
1823
|
}
|
|
1825
1824
|
await writer.commit('1/1');
|
|
1826
|
-
const
|
|
1825
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
1827
1826
|
|
|
1828
1827
|
bucketStorage.clearChecksumCache();
|
|
1829
1828
|
const users = ['u1', 'u2', 'u3', 'u4'];
|
|
1830
1829
|
const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
|
|
1831
1830
|
const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
|
|
1832
|
-
const checksums = [
|
|
1833
|
-
...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), bucketRequests)).values()
|
|
1834
|
-
];
|
|
1831
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
|
|
1835
1832
|
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1836
1833
|
const expected = bucketRequests.map((request, index) => ({
|
|
1837
1834
|
bucket: request.bucket,
|