@powersync/service-core-tests 0.17.2 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/dist/test-utils/AbstractStreamTestContext.d.ts +1 -1
  3. package/dist/test-utils/AbstractStreamTestContext.js +2 -3
  4. package/dist/test-utils/AbstractStreamTestContext.js.map +1 -1
  5. package/dist/test-utils/StorageDataHelpers.d.ts +3 -2
  6. package/dist/test-utils/StorageDataHelpers.js +40 -2
  7. package/dist/test-utils/StorageDataHelpers.js.map +1 -1
  8. package/dist/test-utils/general-utils.d.ts +3 -1
  9. package/dist/test-utils/general-utils.js +13 -1
  10. package/dist/test-utils/general-utils.js.map +1 -1
  11. package/dist/test-utils/stream_utils.d.ts +0 -1
  12. package/dist/test-utils/stream_utils.js +0 -10
  13. package/dist/test-utils/stream_utils.js.map +1 -1
  14. package/dist/tests/register-compacting-tests.js +7 -7
  15. package/dist/tests/register-compacting-tests.js.map +1 -1
  16. package/dist/tests/register-data-storage-checkpoint-tests.js +96 -3
  17. package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
  18. package/dist/tests/register-data-storage-data-tests.js +113 -94
  19. package/dist/tests/register-data-storage-data-tests.js.map +1 -1
  20. package/dist/tests/register-sync-tests.js +259 -75
  21. package/dist/tests/register-sync-tests.js.map +1 -1
  22. package/package.json +4 -4
  23. package/src/test-utils/AbstractStreamTestContext.ts +2 -3
  24. package/src/test-utils/StorageDataHelpers.ts +45 -2
  25. package/src/test-utils/general-utils.ts +27 -3
  26. package/src/test-utils/stream_utils.ts +0 -11
  27. package/src/tests/register-compacting-tests.ts +7 -7
  28. package/src/tests/register-data-storage-checkpoint-tests.ts +122 -3
  29. package/src/tests/register-data-storage-data-tests.ts +129 -132
  30. package/src/tests/register-sync-tests.ts +280 -77
  31. package/tsconfig.tsbuildinfo +1 -1
@@ -52,9 +52,9 @@ bucket_definitions:
52
52
  beforeReplicaId: test_utils.rid('test1')
53
53
  });
54
54
  await writer.commit('1/1');
55
- const { checkpoint } = await bucketStorage.getCheckpoint();
55
+ const checkpoint = await bucketStorage.getCheckpoint();
56
56
  const request = bucketRequest(syncRules, 'global[]');
57
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
57
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
58
58
  const data = batch[0].chunkData.data.map((d) => {
59
59
  return {
60
60
  op: d.op,
@@ -68,9 +68,7 @@ bucket_definitions:
68
68
  { op: 'PUT', object_id: 'test1', checksum: c1 },
69
69
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
70
70
  ]);
71
- const checksums = [
72
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
73
- ];
71
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
74
72
  expect(checksums).toEqual([
75
73
  {
76
74
  bucket: request.bucket,
@@ -107,9 +105,9 @@ bucket_definitions:
107
105
  afterReplicaId: test_utils.rid('test1')
108
106
  });
109
107
  await writer.commit('2/1');
110
- const { checkpoint } = await bucketStorage.getCheckpoint();
108
+ const checkpoint = await bucketStorage.getCheckpoint();
111
109
  const request = bucketRequest(syncRules, 'global[]');
112
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
110
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
113
111
  const data = batch[0].chunkData.data.map((d) => {
114
112
  return {
115
113
  op: d.op,
@@ -119,9 +117,7 @@ bucket_definitions:
119
117
  });
120
118
  const c1 = 2871785649;
121
119
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
122
- const checksums = [
123
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
124
- ];
120
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
125
121
  expect(checksums).toEqual([
126
122
  {
127
123
  bucket: request.bucket,
@@ -163,9 +159,9 @@ bucket_definitions:
163
159
  afterReplicaId: test_utils.rid('test1')
164
160
  });
165
161
  await writer.commit('2/1');
166
- const { checkpoint } = await bucketStorage.getCheckpoint();
162
+ const checkpoint = await bucketStorage.getCheckpoint();
167
163
  const request = bucketRequest(syncRules, 'global[]');
168
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
164
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
169
165
  const data = batch[0].chunkData.data.map((d) => {
170
166
  return {
171
167
  op: d.op,
@@ -175,9 +171,7 @@ bucket_definitions:
175
171
  });
176
172
  const c1 = 2871785649;
177
173
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
178
- const checksums = [
179
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
180
- ];
174
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
181
175
  expect(checksums).toEqual([
182
176
  {
183
177
  bucket: request.bucket,
@@ -215,9 +209,9 @@ bucket_definitions:
215
209
  afterReplicaId: test_utils.rid('test1')
216
210
  });
217
211
  await writer.commit('1/1');
218
- const { checkpoint } = await bucketStorage.getCheckpoint();
212
+ const checkpoint = await bucketStorage.getCheckpoint();
219
213
  const request = bucketRequest(syncRules, 'global[]');
220
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
214
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
221
215
  const data = batch[0].chunkData.data.map((d) => {
222
216
  return {
223
217
  op: d.op,
@@ -227,9 +221,7 @@ bucket_definitions:
227
221
  });
228
222
  const c1 = 2871785649;
229
223
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
230
- const checksums = [
231
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
232
- ];
224
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
233
225
  expect(checksums).toEqual([
234
226
  {
235
227
  bucket: request.bucket,
@@ -289,9 +281,9 @@ bucket_definitions:
289
281
  });
290
282
  await writer.commit('2/1');
291
283
  }
292
- const { checkpoint } = await bucketStorage.getCheckpoint();
284
+ const checkpoint = await bucketStorage.getCheckpoint();
293
285
  const request = bucketRequest(syncRules, 'global[]');
294
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
286
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
295
287
  expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
296
288
  const data = batch[0].chunkData.data.map((d) => {
297
289
  return {
@@ -347,8 +339,8 @@ bucket_definitions:
347
339
  afterReplicaId: test_utils.rid('test2')
348
340
  });
349
341
  await writer.commit('1/1');
350
- const { checkpoint } = await bucketStorage.getCheckpoint();
351
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [bucketRequest(syncRules, 'global[]')]));
342
+ const checkpoint = await bucketStorage.getCheckpoint();
343
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
352
344
  const data = batch[0].chunkData.data.map((d) => {
353
345
  return {
354
346
  op: d.op,
@@ -398,9 +390,9 @@ bucket_definitions:
398
390
  beforeReplicaId: test_utils.rid('test1')
399
391
  });
400
392
  await writer.flush();
401
- const { checkpoint } = await bucketStorage.getCheckpoint();
393
+ const checkpoint = await bucketStorage.getCheckpoint();
402
394
  const request = bucketRequest(syncRules, 'global[]');
403
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
395
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
404
396
  const data = batch[0].chunkData.data.map((d) => {
405
397
  return {
406
398
  op: d.op,
@@ -414,9 +406,7 @@ bucket_definitions:
414
406
  { op: 'PUT', object_id: 'test1', checksum: c1 },
415
407
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
416
408
  ]);
417
- const checksums = [
418
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
419
- ];
409
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
420
410
  expect(checksums).toEqual([
421
411
  {
422
412
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -497,9 +487,9 @@ bucket_definitions:
497
487
  beforeReplicaId: test_utils.rid('test1')
498
488
  });
499
489
  await writer.commit('2/1');
500
- const { checkpoint } = await bucketStorage.getCheckpoint();
490
+ const checkpoint = await bucketStorage.getCheckpoint();
501
491
  const request = bucketRequest(syncRules, 'global[]');
502
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
492
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
503
493
  const data = batch[0].chunkData.data.map((d) => {
504
494
  return {
505
495
  op: d.op,
@@ -515,9 +505,7 @@ bucket_definitions:
515
505
  { op: 'PUT', object_id: 'test1', checksum: c1 },
516
506
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
517
507
  ]);
518
- const checksums = [
519
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
520
- ];
508
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
521
509
  expect(checksums).toEqual([
522
510
  {
523
511
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -639,7 +627,7 @@ bucket_definitions:
639
627
  const result2 = await writer.flush();
640
628
  const checkpoint2 = result2.flushed_op;
641
629
  const request = bucketRequest(syncRules, 'global[]', checkpoint1);
642
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request]));
630
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request]));
643
631
  const data = batch[0].chunkData.data.map((d) => {
644
632
  return {
645
633
  op: d.op,
@@ -725,7 +713,7 @@ bucket_definitions:
725
713
  const result3 = await writer.flush();
726
714
  const checkpoint3 = result3.flushed_op;
727
715
  const request = bucketRequest(syncRules, 'global[]');
728
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
716
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
729
717
  const data = batch[0].chunkData.data.map((d) => {
730
718
  return {
731
719
  op: d.op,
@@ -821,7 +809,7 @@ bucket_definitions:
821
809
  const result3 = await writer.flush();
822
810
  const checkpoint3 = result3.flushed_op;
823
811
  const request = bucketRequest(syncRules, 'global[]');
824
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
812
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }]));
825
813
  const data = batch[0].chunkData.data.map((d) => {
826
814
  return {
827
815
  op: d.op,
@@ -905,12 +893,12 @@ bucket_definitions:
905
893
  afterReplicaId: test_utils.rid('test3')
906
894
  });
907
895
  await writer.commit('1/1');
908
- const { checkpoint } = await bucketStorage.getCheckpoint();
896
+ const checkpoint = await bucketStorage.getCheckpoint();
909
897
  const options = {
910
898
  chunkLimitBytes: 16 * 1024 * 1024
911
899
  };
912
900
  const request = bucketRequest(syncRules, 'global[]');
913
- const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request], options));
901
+ const batch1 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
914
902
  expect(test_utils.getBatchData(batch1)).toEqual([
915
903
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
916
904
  { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
@@ -920,7 +908,7 @@ bucket_definitions:
920
908
  has_more: true,
921
909
  next_after: '2'
922
910
  });
923
- const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
911
+ const batch2 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
924
912
  expect(test_utils.getBatchData(batch2)).toEqual([
925
913
  { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
926
914
  { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
@@ -930,12 +918,13 @@ bucket_definitions:
930
918
  has_more: false,
931
919
  next_after: '4'
932
920
  });
933
- const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
921
+ const batch3 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
934
922
  expect(test_utils.getBatchData(batch3)).toEqual([]);
935
923
  expect(test_utils.getBatchMeta(batch3)).toEqual(null);
936
924
  });
937
925
  test('long batch', async () => {
938
- // Test syncing a batch of data that is limited by count.
926
+ // Test syncing a batch of data that is limited by count. Compressed storage limits persisted
927
+ // documents and finishes yielding all operations from a document once it has been hydrated.
939
928
  await using factory = await generateStorageFactory();
940
929
  const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
941
930
  bucket_definitions:
@@ -961,40 +950,55 @@ bucket_definitions:
961
950
  });
962
951
  }
963
952
  await writer.commit('1/1');
964
- const { checkpoint } = await bucketStorage.getCheckpoint();
953
+ const checkpoint = await bucketStorage.getCheckpoint();
965
954
  const request = bucketRequest(syncRules, 'global[]');
966
- const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request], { limit: 4 }));
967
- expect(test_utils.getBatchData(batch1)).toEqual([
955
+ const batch1 = await test_utils.getSingleBatchItem(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
956
+ const allOperations = [
968
957
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
969
958
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
970
959
  { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
971
- { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
972
- ]);
973
- expect(test_utils.getBatchMeta(batch1)).toEqual({
974
- after: '0',
975
- has_more: true,
976
- next_after: '4'
977
- });
978
- const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
979
- limit: 4
980
- }));
981
- expect(test_utils.getBatchData(batch2)).toEqual([
960
+ { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
982
961
  { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
983
962
  { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
984
- ]);
985
- expect(test_utils.getBatchMeta(batch2)).toEqual({
986
- after: '4',
987
- has_more: false,
988
- next_after: '6'
989
- });
990
- const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
991
- limit: 4
992
- }));
993
- expect(test_utils.getBatchData(batch3)).toEqual([]);
994
- expect(test_utils.getBatchMeta(batch3)).toEqual(null);
963
+ ];
964
+ if (config.compressedBucketStorage) {
965
+ expect(test_utils.getBatchData(batch1)).toEqual(allOperations);
966
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
967
+ after: '0',
968
+ has_more: false,
969
+ next_after: '6'
970
+ });
971
+ const batch2 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
972
+ limit: 4
973
+ }));
974
+ expect(test_utils.getBatchData(batch2)).toEqual([]);
975
+ expect(test_utils.getBatchMeta(batch2)).toEqual(null);
976
+ }
977
+ else {
978
+ expect(test_utils.getBatchData(batch1)).toEqual(allOperations.slice(0, 4));
979
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
980
+ after: '0',
981
+ has_more: true,
982
+ next_after: '4'
983
+ });
984
+ const batch2 = await test_utils.getSingleBatchItem(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
985
+ limit: 4
986
+ }));
987
+ expect(test_utils.getBatchData(batch2)).toEqual(allOperations.slice(4));
988
+ expect(test_utils.getBatchMeta(batch2)).toEqual({
989
+ after: '4',
990
+ has_more: false,
991
+ next_after: '6'
992
+ });
993
+ const batch3 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
994
+ limit: 4
995
+ }));
996
+ expect(test_utils.getBatchData(batch3)).toEqual([]);
997
+ expect(test_utils.getBatchMeta(batch3)).toEqual(null);
998
+ }
995
999
  });
996
1000
  describe('batch has_more', () => {
997
- const setup = async (options) => {
1001
+ const setup = async (options, commitAfter) => {
998
1002
  await using factory = await generateStorageFactory();
999
1003
  const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
1000
1004
  bucket_definitions:
@@ -1020,28 +1024,39 @@ bucket_definitions:
1020
1024
  },
1021
1025
  afterReplicaId: `test${i}`
1022
1026
  });
1027
+ if (i == commitAfter) {
1028
+ await writer.commit('1/1');
1029
+ }
1023
1030
  }
1024
- await writer.commit('1/1');
1025
- const { checkpoint } = await bucketStorage.getCheckpoint();
1031
+ await writer.commit(commitAfter == null ? '1/1' : '1/2');
1032
+ const checkpoint = await bucketStorage.getCheckpoint();
1026
1033
  const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
1027
1034
  const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
1028
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [global1Request, global2Request], options));
1035
+ const batch = await new test_utils.StorageDataHelpers(bucketStorage, syncRules).getAllBucketData([global1Request, global2Request], checkpoint, options);
1029
1036
  return { batch, global1Request, global2Request };
1030
1037
  };
1031
1038
  test('batch has_more (1)', async () => {
1032
- const { batch, global1Request, global2Request } = await setup({ limit: 5 });
1033
- expect(batch.length).toEqual(2);
1039
+ // The commit boundary creates two compressed documents for global2. The limit selects the global1
1040
+ // document and the first global2 document, leaving the second global2 document for the next batch.
1041
+ const { batch, global1Request, global2Request } = await setup({ limit: 2 }, 5);
1042
+ expect(batch.length).toBeGreaterThanOrEqual(2);
1034
1043
  expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1035
1044
  expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1036
1045
  expect(test_utils.getBatchData(batch[0])).toEqual([
1037
1046
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
1038
1047
  ]);
1039
- expect(test_utils.getBatchData(batch[1])).toEqual([
1048
+ const global2Operations = [
1040
1049
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
1041
1050
  { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
1042
1051
  { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
1043
- { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }
1044
- ]);
1052
+ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1053
+ { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
1054
+ { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
1055
+ { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
1056
+ { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
1057
+ { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
1058
+ ];
1059
+ expect(test_utils.getBatchData(batch[1])).toEqual(config.compressedBucketStorage ? global2Operations.slice(0, 4) : global2Operations.slice(0, 1));
1045
1060
  expect(test_utils.getBatchMeta(batch[0])).toEqual({
1046
1061
  after: '0',
1047
1062
  has_more: false,
@@ -1050,7 +1065,7 @@ bucket_definitions:
1050
1065
  expect(test_utils.getBatchMeta(batch[1])).toEqual({
1051
1066
  after: '0',
1052
1067
  has_more: true,
1053
- next_after: '5'
1068
+ next_after: config.compressedBucketStorage ? '5' : '2'
1054
1069
  });
1055
1070
  });
1056
1071
  test('batch has_more (2)', async () => {
@@ -1087,7 +1102,8 @@ bucket_definitions:
1087
1102
  // 50 bytes is more than 1 row, less than 2 rows
1088
1103
  const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
1089
1104
  if (config.compressedBucketStorage) {
1090
- // In v3+, ops in the same bucket share a document, so ops 2 and 3 (global2) are batched together
1105
+ // In v3+, ops in the same bucket share a document. Once hydrated, the entire document is yielded even
1106
+ // when it exceeds the byte limit; the byte limit controls whether another document is fetched.
1091
1107
  expect(batch.length).toEqual(2);
1092
1108
  expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1093
1109
  expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
@@ -1096,7 +1112,14 @@ bucket_definitions:
1096
1112
  ]);
1097
1113
  expect(test_utils.getBatchData(batch[1])).toEqual([
1098
1114
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
1099
- { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
1115
+ { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
1116
+ { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
1117
+ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1118
+ { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
1119
+ { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
1120
+ { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
1121
+ { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
1122
+ { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
1100
1123
  ]);
1101
1124
  expect(test_utils.getBatchMeta(batch[0])).toEqual({
1102
1125
  after: '0',
@@ -1105,12 +1128,12 @@ bucket_definitions:
1105
1128
  });
1106
1129
  expect(test_utils.getBatchMeta(batch[1])).toEqual({
1107
1130
  after: '0',
1108
- has_more: true,
1109
- next_after: '3'
1131
+ has_more: false,
1132
+ next_after: '10'
1110
1133
  });
1111
1134
  }
1112
1135
  else {
1113
- expect(batch.length).toEqual(3);
1136
+ expect(batch.length).toBeGreaterThanOrEqual(3);
1114
1137
  expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1115
1138
  expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1116
1139
  expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
@@ -1225,21 +1248,19 @@ bucket_definitions:
1225
1248
  afterReplicaId: test_utils.rid('test1')
1226
1249
  });
1227
1250
  await writer.commit('1/1');
1228
- const { checkpoint } = await bucketStorage.getCheckpoint();
1251
+ const checkpoint = await bucketStorage.getCheckpoint();
1229
1252
  const request = bucketRequest(syncRules, 'global[]');
1230
- const checksums = [
1231
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
1232
- ];
1253
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
1233
1254
  expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1234
1255
  const checksums2 = [
1235
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint + 1n), [request])).values()
1256
+ ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint.checkpoint + 1n), [request])).values()
1236
1257
  ];
1237
1258
  expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1238
1259
  });
1239
1260
  testChecksumBatching(config);
1240
1261
  test('empty checkpoints (1)', async () => {
1241
1262
  await using factory = await generateStorageFactory();
1242
- const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
1263
+ const { stream: replicationStream } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
1243
1264
  bucket_definitions:
1244
1265
  global:
1245
1266
  data:
@@ -1265,7 +1286,7 @@ bucket_definitions:
1265
1286
  });
1266
1287
  test('empty checkpoints (2)', async () => {
1267
1288
  await using factory = await generateStorageFactory();
1268
- const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
1289
+ const { stream: replicationStream } = await test_utils.deploySyncRules(factory, updateSyncRulesFromYaml(`
1269
1290
  bucket_definitions:
1270
1291
  global:
1271
1292
  data:
@@ -1385,7 +1406,7 @@ bucket_definitions:
1385
1406
  await streamingWriter.keepalive('3/1');
1386
1407
  const cp = await bucketStorage.getCheckpoint();
1387
1408
  expect(cp.lsn).toEqual('3/1');
1388
- const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(cp.checkpoint), [bucketRequest(syncRules, 'global[]')]));
1409
+ const data = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(cp, [bucketRequest(syncRules, 'global[]')]));
1389
1410
  expect(data).toEqual([]);
1390
1411
  });
1391
1412
  }
@@ -1427,14 +1448,12 @@ bucket_definitions:
1427
1448
  }
1428
1449
  }
1429
1450
  await writer.commit('1/1');
1430
- const { checkpoint } = await bucketStorage.getCheckpoint();
1451
+ const checkpoint = await bucketStorage.getCheckpoint();
1431
1452
  bucketStorage.clearChecksumCache();
1432
1453
  const users = ['u1', 'u2', 'u3', 'u4'];
1433
1454
  const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
1434
1455
  const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
1435
- const checksums = [
1436
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), bucketRequests)).values()
1437
- ];
1456
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
1438
1457
  checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
1439
1458
  const expected = bucketRequests.map((request, index) => ({
1440
1459
  bucket: request.bucket,