@powersync/service-core-tests 0.17.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/dist/test-utils/AbstractStreamTestContext.d.ts +1 -1
  3. package/dist/test-utils/AbstractStreamTestContext.js +2 -3
  4. package/dist/test-utils/AbstractStreamTestContext.js.map +1 -1
  5. package/dist/test-utils/StorageDataHelpers.d.ts +3 -2
  6. package/dist/test-utils/StorageDataHelpers.js +40 -2
  7. package/dist/test-utils/StorageDataHelpers.js.map +1 -1
  8. package/dist/test-utils/general-utils.d.ts +3 -1
  9. package/dist/test-utils/general-utils.js +13 -1
  10. package/dist/test-utils/general-utils.js.map +1 -1
  11. package/dist/test-utils/stream_utils.d.ts +0 -1
  12. package/dist/test-utils/stream_utils.js +0 -10
  13. package/dist/test-utils/stream_utils.js.map +1 -1
  14. package/dist/tests/register-compacting-tests.js +7 -7
  15. package/dist/tests/register-compacting-tests.js.map +1 -1
  16. package/dist/tests/register-data-storage-checkpoint-tests.js +96 -3
  17. package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
  18. package/dist/tests/register-data-storage-data-tests.js +113 -94
  19. package/dist/tests/register-data-storage-data-tests.js.map +1 -1
  20. package/dist/tests/register-sync-tests.js +259 -75
  21. package/dist/tests/register-sync-tests.js.map +1 -1
  22. package/package.json +5 -5
  23. package/src/test-utils/AbstractStreamTestContext.ts +2 -3
  24. package/src/test-utils/StorageDataHelpers.ts +45 -2
  25. package/src/test-utils/general-utils.ts +27 -3
  26. package/src/test-utils/stream_utils.ts +0 -11
  27. package/src/tests/register-compacting-tests.ts +7 -7
  28. package/src/tests/register-data-storage-checkpoint-tests.ts +122 -3
  29. package/src/tests/register-data-storage-data-tests.ts +129 -132
  30. package/src/tests/register-sync-tests.ts +280 -77
  31. package/tsconfig.tsbuildinfo +1 -1
@@ -72,12 +72,10 @@ bucket_definitions:
72
72
  });
73
73
  await writer.commit('1/1');
74
74
 
75
- const { checkpoint } = await bucketStorage.getCheckpoint();
75
+ const checkpoint = await bucketStorage.getCheckpoint();
76
76
 
77
77
  const request = bucketRequest(syncRules, 'global[]');
78
- const batch = await test_utils.fromAsync(
79
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
80
- );
78
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
81
79
  const data = batch[0].chunkData.data.map((d) => {
82
80
  return {
83
81
  op: d.op,
@@ -94,9 +92,7 @@ bucket_definitions:
94
92
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
95
93
  ]);
96
94
 
97
- const checksums = [
98
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
99
- ];
95
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
100
96
  expect(checksums).toEqual([
101
97
  {
102
98
  bucket: request.bucket,
@@ -144,12 +140,10 @@ bucket_definitions:
144
140
  });
145
141
  await writer.commit('2/1');
146
142
 
147
- const { checkpoint } = await bucketStorage.getCheckpoint();
143
+ const checkpoint = await bucketStorage.getCheckpoint();
148
144
 
149
145
  const request = bucketRequest(syncRules, 'global[]');
150
- const batch = await test_utils.fromAsync(
151
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
152
- );
146
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
153
147
  const data = batch[0].chunkData.data.map((d) => {
154
148
  return {
155
149
  op: d.op,
@@ -162,9 +156,7 @@ bucket_definitions:
162
156
 
163
157
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
164
158
 
165
- const checksums = [
166
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
167
- ];
159
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
168
160
  expect(checksums).toEqual([
169
161
  {
170
162
  bucket: request.bucket,
@@ -217,12 +209,10 @@ bucket_definitions:
217
209
  });
218
210
  await writer.commit('2/1');
219
211
 
220
- const { checkpoint } = await bucketStorage.getCheckpoint();
212
+ const checkpoint = await bucketStorage.getCheckpoint();
221
213
 
222
214
  const request = bucketRequest(syncRules, 'global[]');
223
- const batch = await test_utils.fromAsync(
224
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
225
- );
215
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
226
216
  const data = batch[0].chunkData.data.map((d) => {
227
217
  return {
228
218
  op: d.op,
@@ -235,9 +225,7 @@ bucket_definitions:
235
225
 
236
226
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
237
227
 
238
- const checksums = [
239
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
240
- ];
228
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
241
229
  expect(checksums).toEqual([
242
230
  {
243
231
  bucket: request.bucket,
@@ -284,12 +272,10 @@ bucket_definitions:
284
272
  });
285
273
  await writer.commit('1/1');
286
274
 
287
- const { checkpoint } = await bucketStorage.getCheckpoint();
275
+ const checkpoint = await bucketStorage.getCheckpoint();
288
276
 
289
277
  const request = bucketRequest(syncRules, 'global[]');
290
- const batch = await test_utils.fromAsync(
291
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
292
- );
278
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
293
279
  const data = batch[0].chunkData.data.map((d) => {
294
280
  return {
295
281
  op: d.op,
@@ -302,9 +288,7 @@ bucket_definitions:
302
288
 
303
289
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
304
290
 
305
- const checksums = [
306
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
307
- ];
291
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
308
292
  expect(checksums).toEqual([
309
293
  {
310
294
  bucket: request.bucket,
@@ -375,12 +359,10 @@ bucket_definitions:
375
359
  await writer.commit('2/1');
376
360
  }
377
361
 
378
- const { checkpoint } = await bucketStorage.getCheckpoint();
362
+ const checkpoint = await bucketStorage.getCheckpoint();
379
363
 
380
364
  const request = bucketRequest(syncRules, 'global[]');
381
- const batch = await test_utils.fromAsync(
382
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
383
- );
365
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
384
366
 
385
367
  expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
386
368
 
@@ -448,9 +430,9 @@ bucket_definitions:
448
430
  });
449
431
 
450
432
  await writer.commit('1/1');
451
- const { checkpoint } = await bucketStorage.getCheckpoint();
452
- const batch = await test_utils.fromAsync(
453
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [bucketRequest(syncRules, 'global[]')])
433
+ const checkpoint = await bucketStorage.getCheckpoint();
434
+ const batch = await test_utils.getBatchArray(
435
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')])
454
436
  );
455
437
  const data = batch[0].chunkData.data.map((d) => {
456
438
  return {
@@ -514,12 +496,10 @@ bucket_definitions:
514
496
  });
515
497
  await writer.flush();
516
498
 
517
- const { checkpoint } = await bucketStorage.getCheckpoint();
499
+ const checkpoint = await bucketStorage.getCheckpoint();
518
500
 
519
501
  const request = bucketRequest(syncRules, 'global[]');
520
- const batch = await test_utils.fromAsync(
521
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
522
- );
502
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
523
503
  const data = batch[0].chunkData.data.map((d) => {
524
504
  return {
525
505
  op: d.op,
@@ -536,9 +516,7 @@ bucket_definitions:
536
516
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
537
517
  ]);
538
518
 
539
- const checksums = [
540
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
541
- ];
519
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
542
520
  expect(checksums).toEqual([
543
521
  {
544
522
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -638,12 +616,10 @@ bucket_definitions:
638
616
 
639
617
  await writer.commit('2/1');
640
618
 
641
- const { checkpoint } = await bucketStorage.getCheckpoint();
619
+ const checkpoint = await bucketStorage.getCheckpoint();
642
620
 
643
621
  const request = bucketRequest(syncRules, 'global[]');
644
- const batch = await test_utils.fromAsync(
645
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
646
- );
622
+ const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
647
623
 
648
624
  const data = batch[0].chunkData.data.map((d) => {
649
625
  return {
@@ -663,9 +639,7 @@ bucket_definitions:
663
639
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
664
640
  ]);
665
641
 
666
- const checksums = [
667
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
668
- ];
642
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
669
643
  expect(checksums).toEqual([
670
644
  {
671
645
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -808,7 +782,7 @@ bucket_definitions:
808
782
  const checkpoint2 = result2!.flushed_op;
809
783
 
810
784
  const request = bucketRequest(syncRules, 'global[]', checkpoint1);
811
- const batch = await test_utils.fromAsync(
785
+ const batch = await test_utils.getBatchArray(
812
786
  bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request])
813
787
  );
814
788
 
@@ -914,7 +888,7 @@ bucket_definitions:
914
888
  const checkpoint3 = result3!.flushed_op;
915
889
 
916
890
  const request = bucketRequest(syncRules, 'global[]');
917
- const batch = await test_utils.fromAsync(
891
+ const batch = await test_utils.getBatchArray(
918
892
  bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }])
919
893
  );
920
894
  const data = batch[0].chunkData.data.map((d) => {
@@ -1028,7 +1002,7 @@ bucket_definitions:
1028
1002
  const checkpoint3 = result3!.flushed_op;
1029
1003
 
1030
1004
  const request = bucketRequest(syncRules, 'global[]');
1031
- const batch = await test_utils.fromAsync(
1005
+ const batch = await test_utils.getBatchArray(
1032
1006
  bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint3), [{ ...request, start: checkpoint1 }])
1033
1007
  );
1034
1008
  const data = batch[0].chunkData.data.map((d) => {
@@ -1129,16 +1103,14 @@ bucket_definitions:
1129
1103
 
1130
1104
  await writer.commit('1/1');
1131
1105
 
1132
- const { checkpoint } = await bucketStorage.getCheckpoint();
1106
+ const checkpoint = await bucketStorage.getCheckpoint();
1133
1107
 
1134
1108
  const options: storage.BucketDataBatchOptions = {
1135
1109
  chunkLimitBytes: 16 * 1024 * 1024
1136
1110
  };
1137
1111
 
1138
1112
  const request = bucketRequest(syncRules, 'global[]');
1139
- const batch1 = await test_utils.fromAsync(
1140
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request], options)
1141
- );
1113
+ const batch1 = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
1142
1114
  expect(test_utils.getBatchData(batch1)).toEqual([
1143
1115
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1144
1116
  { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
@@ -1149,9 +1121,9 @@ bucket_definitions:
1149
1121
  next_after: '2'
1150
1122
  });
1151
1123
 
1152
- const batch2 = await test_utils.fromAsync(
1124
+ const batch2 = await test_utils.getBatchArray(
1153
1125
  bucketStorage.getBucketDataBatch(
1154
- test_utils.testCheckpoint(checkpoint),
1126
+ checkpoint,
1155
1127
  [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }],
1156
1128
  options
1157
1129
  )
@@ -1166,9 +1138,9 @@ bucket_definitions:
1166
1138
  next_after: '4'
1167
1139
  });
1168
1140
 
1169
- const batch3 = await test_utils.fromAsync(
1141
+ const batch3 = await test_utils.getBatchArray(
1170
1142
  bucketStorage.getBucketDataBatch(
1171
- test_utils.testCheckpoint(checkpoint),
1143
+ checkpoint,
1172
1144
  [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }],
1173
1145
  options
1174
1146
  )
@@ -1178,7 +1150,8 @@ bucket_definitions:
1178
1150
  });
1179
1151
 
1180
1152
  test('long batch', async () => {
1181
- // Test syncing a batch of data that is limited by count.
1153
+ // Test syncing a batch of data that is limited by count. Compressed storage limits persisted
1154
+ // documents and finishes yielding all operations from a document once it has been hydrated.
1182
1155
  await using factory = await generateStorageFactory();
1183
1156
  const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
1184
1157
  factory,
@@ -1213,62 +1186,71 @@ bucket_definitions:
1213
1186
 
1214
1187
  await writer.commit('1/1');
1215
1188
 
1216
- const { checkpoint } = await bucketStorage.getCheckpoint();
1189
+ const checkpoint = await bucketStorage.getCheckpoint();
1217
1190
 
1218
1191
  const request = bucketRequest(syncRules, 'global[]');
1219
- const batch1 = await test_utils.oneFromAsync(
1220
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request], { limit: 4 })
1192
+ const batch1 = await test_utils.getSingleBatchItem(
1193
+ bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 })
1221
1194
  );
1222
1195
 
1223
- expect(test_utils.getBatchData(batch1)).toEqual([
1196
+ const allOperations = [
1224
1197
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1225
1198
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
1226
1199
  { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
1227
- { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
1228
- ]);
1200
+ { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
1201
+ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1202
+ { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
1203
+ ];
1229
1204
 
1230
- expect(test_utils.getBatchMeta(batch1)).toEqual({
1231
- after: '0',
1232
- has_more: true,
1233
- next_after: '4'
1234
- });
1205
+ if (config.compressedBucketStorage) {
1206
+ expect(test_utils.getBatchData(batch1)).toEqual(allOperations);
1207
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
1208
+ after: '0',
1209
+ has_more: false,
1210
+ next_after: '6'
1211
+ });
1235
1212
 
1236
- const batch2 = await test_utils.oneFromAsync(
1237
- bucketStorage.getBucketDataBatch(
1238
- test_utils.testCheckpoint(checkpoint),
1239
- [{ ...request, start: BigInt(batch1.chunkData.next_after) }],
1240
- {
1213
+ const batch2 = await test_utils.getBatchArray(
1214
+ bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
1241
1215
  limit: 4
1242
- }
1243
- )
1244
- );
1245
- expect(test_utils.getBatchData(batch2)).toEqual([
1246
- { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1247
- { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
1248
- ]);
1216
+ })
1217
+ );
1218
+ expect(test_utils.getBatchData(batch2)).toEqual([]);
1219
+ expect(test_utils.getBatchMeta(batch2)).toEqual(null);
1220
+ } else {
1221
+ expect(test_utils.getBatchData(batch1)).toEqual(allOperations.slice(0, 4));
1249
1222
 
1250
- expect(test_utils.getBatchMeta(batch2)).toEqual({
1251
- after: '4',
1252
- has_more: false,
1253
- next_after: '6'
1254
- });
1223
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
1224
+ after: '0',
1225
+ has_more: true,
1226
+ next_after: '4'
1227
+ });
1255
1228
 
1256
- const batch3 = await test_utils.fromAsync(
1257
- bucketStorage.getBucketDataBatch(
1258
- test_utils.testCheckpoint(checkpoint),
1259
- [{ ...request, start: BigInt(batch2.chunkData.next_after) }],
1260
- {
1229
+ const batch2 = await test_utils.getSingleBatchItem(
1230
+ bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
1261
1231
  limit: 4
1262
- }
1263
- )
1264
- );
1265
- expect(test_utils.getBatchData(batch3)).toEqual([]);
1232
+ })
1233
+ );
1234
+ expect(test_utils.getBatchData(batch2)).toEqual(allOperations.slice(4));
1266
1235
 
1267
- expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1236
+ expect(test_utils.getBatchMeta(batch2)).toEqual({
1237
+ after: '4',
1238
+ has_more: false,
1239
+ next_after: '6'
1240
+ });
1241
+
1242
+ const batch3 = await test_utils.getBatchArray(
1243
+ bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
1244
+ limit: 4
1245
+ })
1246
+ );
1247
+ expect(test_utils.getBatchData(batch3)).toEqual([]);
1248
+ expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1249
+ }
1268
1250
  });
1269
1251
 
1270
1252
  describe('batch has_more', () => {
1271
- const setup = async (options: BucketDataBatchOptions) => {
1253
+ const setup = async (options: BucketDataBatchOptions, commitAfter?: number) => {
1272
1254
  await using factory = await generateStorageFactory();
1273
1255
  const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
1274
1256
  factory,
@@ -1301,27 +1283,30 @@ bucket_definitions:
1301
1283
  },
1302
1284
  afterReplicaId: `test${i}`
1303
1285
  });
1286
+ if (i == commitAfter) {
1287
+ await writer.commit('1/1');
1288
+ }
1304
1289
  }
1305
1290
 
1306
- await writer.commit('1/1');
1291
+ await writer.commit(commitAfter == null ? '1/1' : '1/2');
1307
1292
 
1308
- const { checkpoint } = await bucketStorage.getCheckpoint();
1293
+ const checkpoint = await bucketStorage.getCheckpoint();
1309
1294
  const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
1310
1295
  const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
1311
- const batch = await test_utils.fromAsync(
1312
- bucketStorage.getBucketDataBatch(
1313
- test_utils.testCheckpoint(checkpoint),
1314
- [global1Request, global2Request],
1315
- options
1316
- )
1296
+ const batch = await new test_utils.StorageDataHelpers(bucketStorage, syncRules).getAllBucketData(
1297
+ [global1Request, global2Request],
1298
+ checkpoint,
1299
+ options
1317
1300
  );
1318
1301
 
1319
1302
  return { batch, global1Request, global2Request };
1320
1303
  };
1321
1304
 
1322
1305
  test('batch has_more (1)', async () => {
1323
- const { batch, global1Request, global2Request } = await setup({ limit: 5 });
1324
- expect(batch.length).toEqual(2);
1306
+ // The commit boundary creates two compressed documents for global2. The limit selects the global1
1307
+ // document and the first global2 document, leaving the second global2 document for the next batch.
1308
+ const { batch, global1Request, global2Request } = await setup({ limit: 2 }, 5);
1309
+ expect(batch.length).toBeGreaterThanOrEqual(2);
1325
1310
 
1326
1311
  expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1327
1312
  expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
@@ -1330,12 +1315,20 @@ bucket_definitions:
1330
1315
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
1331
1316
  ]);
1332
1317
 
1333
- expect(test_utils.getBatchData(batch[1])).toEqual([
1318
+ const global2Operations = [
1334
1319
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
1335
1320
  { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
1336
1321
  { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
1337
- { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }
1338
- ]);
1322
+ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1323
+ { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
1324
+ { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
1325
+ { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
1326
+ { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
1327
+ { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
1328
+ ];
1329
+ expect(test_utils.getBatchData(batch[1])).toEqual(
1330
+ config.compressedBucketStorage ? global2Operations.slice(0, 4) : global2Operations.slice(0, 1)
1331
+ );
1339
1332
 
1340
1333
  expect(test_utils.getBatchMeta(batch[0])).toEqual({
1341
1334
  after: '0',
@@ -1346,7 +1339,7 @@ bucket_definitions:
1346
1339
  expect(test_utils.getBatchMeta(batch[1])).toEqual({
1347
1340
  after: '0',
1348
1341
  has_more: true,
1349
- next_after: '5'
1342
+ next_after: config.compressedBucketStorage ? '5' : '2'
1350
1343
  });
1351
1344
  });
1352
1345
 
@@ -1391,7 +1384,8 @@ bucket_definitions:
1391
1384
  const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
1392
1385
 
1393
1386
  if (config.compressedBucketStorage) {
1394
- // In v3+, ops in the same bucket share a document, so ops 2 and 3 (global2) are batched together
1387
+ // In v3+, ops in the same bucket share a document. Once hydrated, the entire document is yielded even
1388
+ // when it exceeds the byte limit; the byte limit controls whether another document is fetched.
1395
1389
  expect(batch.length).toEqual(2);
1396
1390
  expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1397
1391
  expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
@@ -1402,7 +1396,14 @@ bucket_definitions:
1402
1396
 
1403
1397
  expect(test_utils.getBatchData(batch[1])).toEqual([
1404
1398
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
1405
- { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
1399
+ { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
1400
+ { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 },
1401
+ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1402
+ { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 },
1403
+ { op_id: '7', op: 'PUT', object_id: 'test7', checksum: 2477637855 },
1404
+ { op_id: '8', op: 'PUT', object_id: 'test8', checksum: 3644033632 },
1405
+ { op_id: '9', op: 'PUT', object_id: 'test9', checksum: 1011055869 },
1406
+ { op_id: '10', op: 'PUT', object_id: 'test10', checksum: 1331456365 }
1406
1407
  ]);
1407
1408
 
1408
1409
  expect(test_utils.getBatchMeta(batch[0])).toEqual({
@@ -1413,11 +1414,11 @@ bucket_definitions:
1413
1414
 
1414
1415
  expect(test_utils.getBatchMeta(batch[1])).toEqual({
1415
1416
  after: '0',
1416
- has_more: true,
1417
- next_after: '3'
1417
+ has_more: false,
1418
+ next_after: '10'
1418
1419
  });
1419
1420
  } else {
1420
- expect(batch.length).toEqual(3);
1421
+ expect(batch.length).toBeGreaterThanOrEqual(3);
1421
1422
  expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1422
1423
  expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1423
1424
  expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
@@ -1560,15 +1561,13 @@ bucket_definitions:
1560
1561
  afterReplicaId: test_utils.rid('test1')
1561
1562
  });
1562
1563
  await writer.commit('1/1');
1563
- const { checkpoint } = await bucketStorage.getCheckpoint();
1564
+ const checkpoint = await bucketStorage.getCheckpoint();
1564
1565
 
1565
1566
  const request = bucketRequest(syncRules, 'global[]');
1566
- const checksums = [
1567
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request])).values()
1568
- ];
1567
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
1569
1568
  expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1570
1569
  const checksums2 = [
1571
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint + 1n), [request])).values()
1570
+ ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint.checkpoint + 1n), [request])).values()
1572
1571
  ];
1573
1572
  expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1574
1573
  });
@@ -1577,7 +1576,7 @@ bucket_definitions:
1577
1576
 
1578
1577
  test('empty checkpoints (1)', async () => {
1579
1578
  await using factory = await generateStorageFactory();
1580
- const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
1579
+ const { stream: replicationStream } = await test_utils.deploySyncRules(
1581
1580
  factory,
1582
1581
  updateSyncRulesFromYaml(
1583
1582
  `
@@ -1614,7 +1613,7 @@ bucket_definitions:
1614
1613
 
1615
1614
  test('empty checkpoints (2)', async () => {
1616
1615
  await using factory = await generateStorageFactory();
1617
- const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules(
1616
+ const { stream: replicationStream } = await test_utils.deploySyncRules(
1618
1617
  factory,
1619
1618
  updateSyncRulesFromYaml(
1620
1619
  `
@@ -1771,8 +1770,8 @@ bucket_definitions:
1771
1770
 
1772
1771
  const cp = await bucketStorage.getCheckpoint();
1773
1772
  expect(cp.lsn).toEqual('3/1');
1774
- const data = await test_utils.fromAsync(
1775
- bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(cp.checkpoint), [bucketRequest(syncRules, 'global[]')])
1773
+ const data = await test_utils.getBatchArray(
1774
+ bucketStorage.getBucketDataBatch(cp, [bucketRequest(syncRules, 'global[]')])
1776
1775
  );
1777
1776
 
1778
1777
  expect(data).toEqual([]);
@@ -1823,15 +1822,13 @@ bucket_definitions:
1823
1822
  }
1824
1823
  }
1825
1824
  await writer.commit('1/1');
1826
- const { checkpoint } = await bucketStorage.getCheckpoint();
1825
+ const checkpoint = await bucketStorage.getCheckpoint();
1827
1826
 
1828
1827
  bucketStorage.clearChecksumCache();
1829
1828
  const users = ['u1', 'u2', 'u3', 'u4'];
1830
1829
  const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
1831
1830
  const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
1832
- const checksums = [
1833
- ...(await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), bucketRequests)).values()
1834
- ];
1831
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
1835
1832
  checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
1836
1833
  const expected = bucketRequests.map((request, index) => ({
1837
1834
  bucket: request.bucket,