@powersync/service-core-tests 0.15.0 → 0.15.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/dist/test-utils/general-utils.d.ts +13 -1
  3. package/dist/test-utils/general-utils.js +30 -1
  4. package/dist/test-utils/general-utils.js.map +1 -1
  5. package/dist/test-utils/stream_utils.js +2 -2
  6. package/dist/test-utils/stream_utils.js.map +1 -1
  7. package/dist/tests/register-compacting-tests.js +266 -257
  8. package/dist/tests/register-compacting-tests.js.map +1 -1
  9. package/dist/tests/register-data-storage-checkpoint-tests.js +36 -57
  10. package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
  11. package/dist/tests/register-data-storage-data-tests.js +839 -863
  12. package/dist/tests/register-data-storage-data-tests.js.map +1 -1
  13. package/dist/tests/register-data-storage-parameter-tests.js +228 -236
  14. package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
  15. package/dist/tests/register-parameter-compacting-tests.js +81 -89
  16. package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
  17. package/dist/tests/register-sync-tests.js +468 -462
  18. package/dist/tests/register-sync-tests.js.map +1 -1
  19. package/package.json +3 -3
  20. package/src/test-utils/general-utils.ts +41 -2
  21. package/src/test-utils/stream_utils.ts +2 -2
  22. package/src/tests/register-compacting-tests.ts +279 -270
  23. package/src/tests/register-data-storage-checkpoint-tests.ts +36 -57
  24. package/src/tests/register-data-storage-data-tests.ts +673 -770
  25. package/src/tests/register-data-storage-parameter-tests.ts +245 -257
  26. package/src/tests/register-parameter-compacting-tests.ts +84 -92
  27. package/src/tests/register-sync-tests.ts +375 -391
  28. package/tsconfig.tsbuildinfo +1 -1
@@ -10,7 +10,6 @@ import {
10
10
  import { describe, expect, test } from 'vitest';
11
11
  import * as test_utils from '../test-utils/test-utils-index.js';
12
12
  import { bucketRequest } from '../test-utils/test-utils-index.js';
13
- import { bucketRequestMap, bucketRequests } from './util.js';
14
13
 
15
14
  /**
16
15
  * Normalize data from OplogEntries for comparison in tests.
@@ -37,8 +36,6 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig)
37
36
  const generateStorageFactory = config.factory;
38
37
  const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
39
38
 
40
- const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
41
-
42
39
  test('removing row', async () => {
43
40
  await using factory = await generateStorageFactory();
44
41
  const syncRules = await factory.updateSyncRules(
@@ -53,33 +50,31 @@ bucket_definitions:
53
50
  )
54
51
  );
55
52
  const bucketStorage = factory.getInstance(syncRules);
53
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
54
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
56
55
 
57
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
58
- const sourceTable = TEST_TABLE;
59
- await batch.markAllSnapshotDone('1/1');
56
+ await writer.markAllSnapshotDone('1/1');
60
57
 
61
- await batch.save({
62
- sourceTable,
63
- tag: storage.SaveOperationTag.INSERT,
64
- after: {
65
- id: 'test1',
66
- description: 'test1'
67
- },
68
- afterReplicaId: test_utils.rid('test1')
69
- });
70
- await batch.save({
71
- sourceTable,
72
- tag: storage.SaveOperationTag.DELETE,
73
- beforeReplicaId: test_utils.rid('test1')
74
- });
75
- await batch.commit('1/1');
58
+ await writer.save({
59
+ sourceTable: testTable,
60
+ tag: storage.SaveOperationTag.INSERT,
61
+ after: {
62
+ id: 'test1',
63
+ description: 'test1'
64
+ },
65
+ afterReplicaId: test_utils.rid('test1')
76
66
  });
67
+ await writer.save({
68
+ sourceTable: testTable,
69
+ tag: storage.SaveOperationTag.DELETE,
70
+ beforeReplicaId: test_utils.rid('test1')
71
+ });
72
+ await writer.commit('1/1');
77
73
 
78
74
  const { checkpoint } = await bucketStorage.getCheckpoint();
79
75
 
80
- const batch = await test_utils.fromAsync(
81
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
82
- );
76
+ const request = bucketRequest(syncRules, 'global[]');
77
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
83
78
  const data = batch[0].chunkData.data.map((d) => {
84
79
  return {
85
80
  op: d.op,
@@ -96,12 +91,10 @@ bucket_definitions:
96
91
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
97
92
  ]);
98
93
 
99
- const checksums = [
100
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
101
- ];
94
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
102
95
  expect(checksums).toEqual([
103
96
  {
104
- bucket: bucketRequest(syncRules, 'global[]').bucket,
97
+ bucket: request.bucket,
105
98
  checksum: (c1 + c2) & 0xffffffff,
106
99
  count: 2
107
100
  }
@@ -122,40 +115,33 @@ bucket_definitions:
122
115
  )
123
116
  );
124
117
  const bucketStorage = factory.getInstance(syncRules);
125
-
126
- const sourceTable = TEST_TABLE;
127
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
128
- await batch.markAllSnapshotDone('1/1');
129
-
130
- await batch.save({
131
- sourceTable,
132
- tag: storage.SaveOperationTag.DELETE,
133
- beforeReplicaId: test_utils.rid('test1')
134
- });
135
-
136
- await batch.commit('0/1');
118
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
119
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
120
+ await writer.markAllSnapshotDone('1/1');
121
+
122
+ await writer.save({
123
+ sourceTable: testTable,
124
+ tag: storage.SaveOperationTag.DELETE,
125
+ beforeReplicaId: test_utils.rid('test1')
137
126
  });
138
127
 
139
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
140
- const sourceTable = TEST_TABLE;
128
+ await writer.commit('0/1');
141
129
 
142
- await batch.save({
143
- sourceTable,
144
- tag: storage.SaveOperationTag.INSERT,
145
- after: {
146
- id: 'test1',
147
- description: 'test1'
148
- },
149
- afterReplicaId: test_utils.rid('test1')
150
- });
151
- await batch.commit('2/1');
130
+ await writer.save({
131
+ sourceTable: testTable,
132
+ tag: storage.SaveOperationTag.INSERT,
133
+ after: {
134
+ id: 'test1',
135
+ description: 'test1'
136
+ },
137
+ afterReplicaId: test_utils.rid('test1')
152
138
  });
139
+ await writer.commit('2/1');
153
140
 
154
141
  const { checkpoint } = await bucketStorage.getCheckpoint();
155
142
 
156
- const batch = await test_utils.fromAsync(
157
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
158
- );
143
+ const request = bucketRequest(syncRules, 'global[]');
144
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
159
145
  const data = batch[0].chunkData.data.map((d) => {
160
146
  return {
161
147
  op: d.op,
@@ -168,12 +154,10 @@ bucket_definitions:
168
154
 
169
155
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
170
156
 
171
- const checksums = [
172
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
173
- ];
157
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
174
158
  expect(checksums).toEqual([
175
159
  {
176
- bucket: bucketRequest(syncRules, 'global[]').bucket,
160
+ bucket: request.bucket,
177
161
  checksum: c1 & 0xffffffff,
178
162
  count: 1
179
163
  }
@@ -195,44 +179,37 @@ bucket_definitions:
195
179
  )
196
180
  );
197
181
  const bucketStorage = factory.getInstance(syncRules);
198
-
199
- const sourceTable = TEST_TABLE;
200
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
201
- await batch.markAllSnapshotDone('1/1');
202
-
203
- await batch.save({
204
- sourceTable,
205
- tag: storage.SaveOperationTag.DELETE,
206
- beforeReplicaId: test_utils.rid('test1')
207
- });
208
-
209
- await batch.commit('0/1');
182
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
183
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
184
+ await writer.markAllSnapshotDone('1/1');
185
+
186
+ await writer.save({
187
+ sourceTable: testTable,
188
+ tag: storage.SaveOperationTag.DELETE,
189
+ beforeReplicaId: test_utils.rid('test1')
210
190
  });
211
191
 
212
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
213
- const sourceTable = TEST_TABLE;
192
+ await writer.commit('0/1');
214
193
 
215
- await batch.save({
216
- sourceTable,
217
- tag: storage.SaveOperationTag.UPDATE,
218
- before: {
219
- id: 'test1'
220
- },
221
- after: {
222
- id: 'test1',
223
- description: 'test1'
224
- },
225
- beforeReplicaId: test_utils.rid('test1'),
226
- afterReplicaId: test_utils.rid('test1')
227
- });
228
- await batch.commit('2/1');
194
+ await writer.save({
195
+ sourceTable: testTable,
196
+ tag: storage.SaveOperationTag.UPDATE,
197
+ before: {
198
+ id: 'test1'
199
+ },
200
+ after: {
201
+ id: 'test1',
202
+ description: 'test1'
203
+ },
204
+ beforeReplicaId: test_utils.rid('test1'),
205
+ afterReplicaId: test_utils.rid('test1')
229
206
  });
207
+ await writer.commit('2/1');
230
208
 
231
209
  const { checkpoint } = await bucketStorage.getCheckpoint();
232
210
 
233
- const batch = await test_utils.fromAsync(
234
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
235
- );
211
+ const request = bucketRequest(syncRules, 'global[]');
212
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
236
213
  const data = batch[0].chunkData.data.map((d) => {
237
214
  return {
238
215
  op: d.op,
@@ -245,12 +222,10 @@ bucket_definitions:
245
222
 
246
223
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
247
224
 
248
- const checksums = [
249
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
250
- ];
225
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
251
226
  expect(checksums).toEqual([
252
227
  {
253
- bucket: bucketRequest(syncRules, 'global[]').bucket,
228
+ bucket: request.bucket,
254
229
  checksum: c1 & 0xffffffff,
255
230
  count: 1
256
231
  }
@@ -273,33 +248,30 @@ bucket_definitions:
273
248
  )
274
249
  );
275
250
  const bucketStorage = factory.getInstance(syncRules);
276
-
277
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
278
- const sourceTable = TEST_TABLE;
279
- await batch.markAllSnapshotDone('1/1');
280
-
281
- await batch.save({
282
- sourceTable,
283
- tag: storage.SaveOperationTag.DELETE,
284
- beforeReplicaId: test_utils.rid('test1')
285
- });
286
- await batch.save({
287
- sourceTable,
288
- tag: storage.SaveOperationTag.INSERT,
289
- after: {
290
- id: 'test1',
291
- description: 'test1'
292
- },
293
- afterReplicaId: test_utils.rid('test1')
294
- });
295
- await batch.commit('1/1');
251
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
252
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
253
+ await writer.markAllSnapshotDone('1/1');
254
+
255
+ await writer.save({
256
+ sourceTable: testTable,
257
+ tag: storage.SaveOperationTag.DELETE,
258
+ beforeReplicaId: test_utils.rid('test1')
259
+ });
260
+ await writer.save({
261
+ sourceTable: testTable,
262
+ tag: storage.SaveOperationTag.INSERT,
263
+ after: {
264
+ id: 'test1',
265
+ description: 'test1'
266
+ },
267
+ afterReplicaId: test_utils.rid('test1')
296
268
  });
269
+ await writer.commit('1/1');
297
270
 
298
271
  const { checkpoint } = await bucketStorage.getCheckpoint();
299
272
 
300
- const batch = await test_utils.fromAsync(
301
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
302
- );
273
+ const request = bucketRequest(syncRules, 'global[]');
274
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
303
275
  const data = batch[0].chunkData.data.map((d) => {
304
276
  return {
305
277
  op: d.op,
@@ -312,12 +284,10 @@ bucket_definitions:
312
284
 
313
285
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
314
286
 
315
- const checksums = [
316
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
317
- ];
287
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
318
288
  expect(checksums).toEqual([
319
289
  {
320
- bucket: bucketRequest(syncRules, 'global[]').bucket,
290
+ bucket: request.bucket,
321
291
  checksum: c1 & 0xffffffff,
322
292
  count: 1
323
293
  }
@@ -340,12 +310,12 @@ bucket_definitions:
340
310
  )
341
311
  );
342
312
  const bucketStorage = factory.getInstance(syncRules);
313
+ {
314
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
315
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
316
+ await writer.markAllSnapshotDone('1/1');
343
317
 
344
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
345
- const sourceTable = TEST_TABLE;
346
- await batch.markAllSnapshotDone('1/1');
347
-
348
- await batch.save({
318
+ await writer.save({
349
319
  sourceTable,
350
320
  tag: storage.SaveOperationTag.INSERT,
351
321
  after: {
@@ -354,12 +324,12 @@ bucket_definitions:
354
324
  },
355
325
  afterReplicaId: test_utils.rid('test1')
356
326
  });
357
- await batch.save({
327
+ await writer.save({
358
328
  sourceTable,
359
329
  tag: storage.SaveOperationTag.DELETE,
360
330
  beforeReplicaId: test_utils.rid('test1')
361
331
  });
362
- await batch.save({
332
+ await writer.save({
363
333
  sourceTable,
364
334
  tag: storage.SaveOperationTag.INSERT,
365
335
  after: {
@@ -368,26 +338,26 @@ bucket_definitions:
368
338
  },
369
339
  afterReplicaId: test_utils.rid('test1')
370
340
  });
371
- await batch.commit('1/1');
372
- });
341
+ await writer.commit('1/1');
342
+ }
373
343
 
374
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
375
- const sourceTable = TEST_TABLE;
376
- await batch.markAllSnapshotDone('1/1');
344
+ {
345
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
346
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
347
+ await writer.markAllSnapshotDone('1/1');
377
348
 
378
- await batch.save({
349
+ await writer.save({
379
350
  sourceTable,
380
351
  tag: storage.SaveOperationTag.DELETE,
381
352
  beforeReplicaId: test_utils.rid('test1')
382
353
  });
383
- await batch.commit('2/1');
384
- });
354
+ await writer.commit('2/1');
355
+ }
385
356
 
386
357
  const { checkpoint } = await bucketStorage.getCheckpoint();
387
358
 
388
- const batch = await test_utils.fromAsync(
389
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
390
- );
359
+ const request = bucketRequest(syncRules, 'global[]');
360
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
391
361
 
392
362
  expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
393
363
 
@@ -418,47 +388,45 @@ bucket_definitions:
418
388
  )
419
389
  );
420
390
  const bucketStorage = factory.getInstance(syncRules);
391
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
392
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
393
+ await writer.markAllSnapshotDone('1/1');
394
+ await writer.save({
395
+ sourceTable,
396
+ tag: storage.SaveOperationTag.INSERT,
397
+ after: {
398
+ id: 'test1',
399
+ client_id: 'client1a',
400
+ description: 'test1a'
401
+ },
402
+ afterReplicaId: test_utils.rid('test1')
403
+ });
404
+ await writer.save({
405
+ sourceTable,
406
+ tag: storage.SaveOperationTag.UPDATE,
407
+ after: {
408
+ id: 'test1',
409
+ client_id: 'client1b',
410
+ description: 'test1b'
411
+ },
412
+ afterReplicaId: test_utils.rid('test1')
413
+ });
421
414
 
422
- const sourceTable = TEST_TABLE;
423
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
424
- await batch.markAllSnapshotDone('1/1');
425
- await batch.save({
426
- sourceTable,
427
- tag: storage.SaveOperationTag.INSERT,
428
- after: {
429
- id: 'test1',
430
- client_id: 'client1a',
431
- description: 'test1a'
432
- },
433
- afterReplicaId: test_utils.rid('test1')
434
- });
435
- await batch.save({
436
- sourceTable,
437
- tag: storage.SaveOperationTag.UPDATE,
438
- after: {
439
- id: 'test1',
440
- client_id: 'client1b',
441
- description: 'test1b'
442
- },
443
- afterReplicaId: test_utils.rid('test1')
444
- });
445
-
446
- await batch.save({
447
- sourceTable,
448
- tag: storage.SaveOperationTag.INSERT,
449
- after: {
450
- id: 'test2',
451
- client_id: 'client2',
452
- description: 'test2'
453
- },
454
- afterReplicaId: test_utils.rid('test2')
455
- });
456
-
457
- await batch.commit('1/1');
415
+ await writer.save({
416
+ sourceTable,
417
+ tag: storage.SaveOperationTag.INSERT,
418
+ after: {
419
+ id: 'test2',
420
+ client_id: 'client2',
421
+ description: 'test2'
422
+ },
423
+ afterReplicaId: test_utils.rid('test2')
458
424
  });
425
+
426
+ await writer.commit('1/1');
459
427
  const { checkpoint } = await bucketStorage.getCheckpoint();
460
428
  const batch = await test_utils.fromAsync(
461
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
429
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')])
462
430
  );
463
431
  const data = batch[0].chunkData.data.map((d) => {
464
432
  return {
@@ -491,49 +459,40 @@ bucket_definitions:
491
459
  )
492
460
  );
493
461
  const bucketStorage = factory.getInstance(syncRules);
494
-
495
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
496
- const sourceTable = TEST_TABLE;
497
- await batch.markAllSnapshotDone('1/1');
498
-
499
- await batch.save({
500
- sourceTable,
501
- tag: storage.SaveOperationTag.INSERT,
502
- after: {
503
- id: 'test1',
504
- description: 'test1'
505
- },
506
- afterReplicaId: test_utils.rid('test1')
507
- });
462
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
463
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
464
+ await writer.markAllSnapshotDone('1/1');
465
+
466
+ await writer.save({
467
+ sourceTable,
468
+ tag: storage.SaveOperationTag.INSERT,
469
+ after: {
470
+ id: 'test1',
471
+ description: 'test1'
472
+ },
473
+ afterReplicaId: test_utils.rid('test1')
508
474
  });
475
+ await writer.flush();
509
476
 
510
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
511
- const sourceTable = TEST_TABLE;
512
-
513
- await batch.save({
514
- sourceTable,
515
- tag: storage.SaveOperationTag.DELETE,
516
- beforeReplicaId: test_utils.rid('test1')
517
- });
518
-
519
- await batch.commit('1/1');
477
+ await writer.save({
478
+ sourceTable,
479
+ tag: storage.SaveOperationTag.DELETE,
480
+ beforeReplicaId: test_utils.rid('test1')
520
481
  });
521
482
 
522
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
523
- const sourceTable = TEST_TABLE;
483
+ await writer.commit('1/1');
524
484
 
525
- await batch.save({
526
- sourceTable,
527
- tag: storage.SaveOperationTag.DELETE,
528
- beforeReplicaId: test_utils.rid('test1')
529
- });
485
+ await writer.save({
486
+ sourceTable,
487
+ tag: storage.SaveOperationTag.DELETE,
488
+ beforeReplicaId: test_utils.rid('test1')
530
489
  });
490
+ await writer.flush();
531
491
 
532
492
  const { checkpoint } = await bucketStorage.getCheckpoint();
533
493
 
534
- const batch = await test_utils.fromAsync(
535
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
536
- );
494
+ const request = bucketRequest(syncRules, 'global[]');
495
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
537
496
  const data = batch[0].chunkData.data.map((d) => {
538
497
  return {
539
498
  op: d.op,
@@ -550,9 +509,7 @@ bucket_definitions:
550
509
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
551
510
  ]);
552
511
 
553
- const checksums = [
554
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
555
- ];
512
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
556
513
  expect(checksums).toEqual([
557
514
  {
558
515
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -576,93 +533,85 @@ bucket_definitions:
576
533
  )
577
534
  );
578
535
  const bucketStorage = factory.getInstance(syncRules);
579
-
580
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
581
- await batch.markAllSnapshotDone('1/1');
582
- const sourceTable = TEST_TABLE;
583
-
584
- await batch.save({
585
- sourceTable,
586
- tag: storage.SaveOperationTag.INSERT,
587
- after: {
588
- id: 'test1',
589
- description: 'test1'
590
- },
591
- afterReplicaId: test_utils.rid('test1')
592
- });
536
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
537
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
538
+ await writer.markAllSnapshotDone('1/1');
539
+
540
+ await writer.save({
541
+ sourceTable,
542
+ tag: storage.SaveOperationTag.INSERT,
543
+ after: {
544
+ id: 'test1',
545
+ description: 'test1'
546
+ },
547
+ afterReplicaId: test_utils.rid('test1')
593
548
  });
549
+ await writer.flush();
594
550
 
595
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
596
- await batch.markAllSnapshotDone('1/1');
597
- const sourceTable = TEST_TABLE;
598
-
599
- await batch.save({
600
- sourceTable,
601
- tag: storage.SaveOperationTag.UPDATE,
602
- after: {
603
- id: 'test1',
604
- description: undefined
605
- },
606
- afterReplicaId: test_utils.rid('test1')
607
- });
551
+ await writer.markAllSnapshotDone('1/1');
608
552
 
609
- await batch.save({
610
- sourceTable,
611
- tag: storage.SaveOperationTag.UPDATE,
612
- after: {
613
- id: 'test1',
614
- description: undefined
615
- },
616
- afterReplicaId: test_utils.rid('test1')
617
- });
553
+ await writer.save({
554
+ sourceTable,
555
+ tag: storage.SaveOperationTag.UPDATE,
556
+ after: {
557
+ id: 'test1',
558
+ description: undefined
559
+ },
560
+ afterReplicaId: test_utils.rid('test1')
561
+ });
618
562
 
619
- await batch.save({
620
- sourceTable,
621
- tag: storage.SaveOperationTag.DELETE,
622
- beforeReplicaId: test_utils.rid('test1')
623
- });
563
+ await writer.save({
564
+ sourceTable,
565
+ tag: storage.SaveOperationTag.UPDATE,
566
+ after: {
567
+ id: 'test1',
568
+ description: undefined
569
+ },
570
+ afterReplicaId: test_utils.rid('test1')
571
+ });
624
572
 
625
- await batch.commit('1/1');
573
+ await writer.save({
574
+ sourceTable,
575
+ tag: storage.SaveOperationTag.DELETE,
576
+ beforeReplicaId: test_utils.rid('test1')
626
577
  });
627
578
 
628
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
629
- await batch.markAllSnapshotDone('1/1');
630
- const sourceTable = TEST_TABLE;
579
+ await writer.commit('1/1');
631
580
 
632
- await batch.save({
633
- sourceTable,
634
- tag: storage.SaveOperationTag.UPDATE,
635
- after: {
636
- id: 'test1',
637
- description: undefined
638
- },
639
- afterReplicaId: test_utils.rid('test1')
640
- });
581
+ await writer.markAllSnapshotDone('1/1');
641
582
 
642
- await batch.save({
643
- sourceTable,
644
- tag: storage.SaveOperationTag.UPDATE,
645
- after: {
646
- id: 'test1',
647
- description: undefined
648
- },
649
- afterReplicaId: test_utils.rid('test1')
650
- });
583
+ await writer.save({
584
+ sourceTable,
585
+ tag: storage.SaveOperationTag.UPDATE,
586
+ after: {
587
+ id: 'test1',
588
+ description: undefined
589
+ },
590
+ afterReplicaId: test_utils.rid('test1')
591
+ });
651
592
 
652
- await batch.save({
653
- sourceTable,
654
- tag: storage.SaveOperationTag.DELETE,
655
- beforeReplicaId: test_utils.rid('test1')
656
- });
593
+ await writer.save({
594
+ sourceTable,
595
+ tag: storage.SaveOperationTag.UPDATE,
596
+ after: {
597
+ id: 'test1',
598
+ description: undefined
599
+ },
600
+ afterReplicaId: test_utils.rid('test1')
601
+ });
657
602
 
658
- await batch.commit('2/1');
603
+ await writer.save({
604
+ sourceTable,
605
+ tag: storage.SaveOperationTag.DELETE,
606
+ beforeReplicaId: test_utils.rid('test1')
659
607
  });
660
608
 
609
+ await writer.commit('2/1');
610
+
661
611
  const { checkpoint } = await bucketStorage.getCheckpoint();
662
612
 
663
- const batch = await test_utils.fromAsync(
664
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
665
- );
613
+ const request = bucketRequest(syncRules, 'global[]');
614
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
666
615
 
667
616
  const data = batch[0].chunkData.data.map((d) => {
668
617
  return {
@@ -682,9 +631,7 @@ bucket_definitions:
682
631
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
683
632
  ]);
684
633
 
685
- const checksums = [
686
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
687
- ];
634
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
688
635
  expect(checksums).toEqual([
689
636
  {
690
637
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -716,120 +663,117 @@ bucket_definitions:
716
663
  )
717
664
  );
718
665
  const bucketStorage = factory.getInstance(syncRules);
666
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
667
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
719
668
 
720
669
  // Pre-setup
721
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
722
- await batch.markAllSnapshotDone('1/1');
723
- const sourceTable = TEST_TABLE;
724
-
725
- await batch.save({
726
- sourceTable,
727
- tag: storage.SaveOperationTag.INSERT,
728
- after: {
729
- id: 'test1',
730
- description: 'test1a'
731
- },
732
- afterReplicaId: test_utils.rid('test1')
733
- });
670
+ await writer.markAllSnapshotDone('1/1');
671
+
672
+ await writer.save({
673
+ sourceTable,
674
+ tag: storage.SaveOperationTag.INSERT,
675
+ after: {
676
+ id: 'test1',
677
+ description: 'test1a'
678
+ },
679
+ afterReplicaId: test_utils.rid('test1')
680
+ });
734
681
 
735
- await batch.save({
736
- sourceTable,
737
- tag: storage.SaveOperationTag.INSERT,
738
- after: {
739
- id: 'test2',
740
- description: 'test2a'
741
- },
742
- afterReplicaId: test_utils.rid('test2')
743
- });
682
+ await writer.save({
683
+ sourceTable,
684
+ tag: storage.SaveOperationTag.INSERT,
685
+ after: {
686
+ id: 'test2',
687
+ description: 'test2a'
688
+ },
689
+ afterReplicaId: test_utils.rid('test2')
744
690
  });
691
+ const result1 = await writer.flush();
745
692
 
746
693
  const checkpoint1 = result1?.flushed_op ?? 0n;
747
694
 
748
695
  // Test batch
749
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
750
- const sourceTable = TEST_TABLE;
751
- // b
752
- await batch.save({
753
- sourceTable,
754
- tag: storage.SaveOperationTag.INSERT,
755
- after: {
756
- id: 'test1',
757
- description: 'test1b'
758
- },
759
- afterReplicaId: test_utils.rid('test1')
760
- });
696
+ // b
697
+ await writer.save({
698
+ sourceTable,
699
+ tag: storage.SaveOperationTag.INSERT,
700
+ after: {
701
+ id: 'test1',
702
+ description: 'test1b'
703
+ },
704
+ afterReplicaId: test_utils.rid('test1')
705
+ });
761
706
 
762
- await batch.save({
763
- sourceTable,
764
- tag: storage.SaveOperationTag.UPDATE,
765
- before: {
766
- id: 'test1'
767
- },
768
- beforeReplicaId: test_utils.rid('test1'),
769
- after: {
770
- id: 'test2',
771
- description: 'test2b'
772
- },
773
- afterReplicaId: test_utils.rid('test2')
774
- });
707
+ await writer.save({
708
+ sourceTable,
709
+ tag: storage.SaveOperationTag.UPDATE,
710
+ before: {
711
+ id: 'test1'
712
+ },
713
+ beforeReplicaId: test_utils.rid('test1'),
714
+ after: {
715
+ id: 'test2',
716
+ description: 'test2b'
717
+ },
718
+ afterReplicaId: test_utils.rid('test2')
719
+ });
775
720
 
776
- await batch.save({
777
- sourceTable,
778
- tag: storage.SaveOperationTag.UPDATE,
779
- before: {
780
- id: 'test2'
781
- },
782
- beforeReplicaId: test_utils.rid('test2'),
783
- after: {
784
- id: 'test3',
785
- description: 'test3b'
786
- },
721
+ await writer.save({
722
+ sourceTable,
723
+ tag: storage.SaveOperationTag.UPDATE,
724
+ before: {
725
+ id: 'test2'
726
+ },
727
+ beforeReplicaId: test_utils.rid('test2'),
728
+ after: {
729
+ id: 'test3',
730
+ description: 'test3b'
731
+ },
787
732
 
788
- afterReplicaId: test_utils.rid('test3')
789
- });
733
+ afterReplicaId: test_utils.rid('test3')
734
+ });
790
735
 
791
- // c
792
- await batch.save({
793
- sourceTable,
794
- tag: storage.SaveOperationTag.UPDATE,
795
- after: {
796
- id: 'test2',
797
- description: 'test2c'
798
- },
799
- afterReplicaId: test_utils.rid('test2')
800
- });
736
+ // c
737
+ await writer.save({
738
+ sourceTable,
739
+ tag: storage.SaveOperationTag.UPDATE,
740
+ after: {
741
+ id: 'test2',
742
+ description: 'test2c'
743
+ },
744
+ afterReplicaId: test_utils.rid('test2')
745
+ });
801
746
 
802
- // d
803
- await batch.save({
804
- sourceTable,
805
- tag: storage.SaveOperationTag.INSERT,
806
- after: {
807
- id: 'test4',
808
- description: 'test4d'
809
- },
810
- afterReplicaId: test_utils.rid('test4')
811
- });
747
+ // d
748
+ await writer.save({
749
+ sourceTable,
750
+ tag: storage.SaveOperationTag.INSERT,
751
+ after: {
752
+ id: 'test4',
753
+ description: 'test4d'
754
+ },
755
+ afterReplicaId: test_utils.rid('test4')
756
+ });
812
757
 
813
- await batch.save({
814
- sourceTable,
815
- tag: storage.SaveOperationTag.UPDATE,
816
- before: {
817
- id: 'test4'
818
- },
819
- beforeReplicaId: test_utils.rid('test4'),
820
- after: {
821
- id: 'test5',
822
- description: 'test5d'
823
- },
824
- afterReplicaId: test_utils.rid('test5')
825
- });
758
+ await writer.save({
759
+ sourceTable,
760
+ tag: storage.SaveOperationTag.UPDATE,
761
+ before: {
762
+ id: 'test4'
763
+ },
764
+ beforeReplicaId: test_utils.rid('test4'),
765
+ after: {
766
+ id: 'test5',
767
+ description: 'test5d'
768
+ },
769
+ afterReplicaId: test_utils.rid('test5')
826
770
  });
771
+ const result2 = await writer.flush();
827
772
 
828
773
  const checkpoint2 = result2!.flushed_op;
829
774
 
830
- const batch = await test_utils.fromAsync(
831
- bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', checkpoint1]]))
832
- );
775
+ const request = bucketRequest(syncRules, 'global[]', checkpoint1);
776
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
833
777
 
834
778
  const data = batch[0].chunkData.data.map((d) => {
835
779
  return {
@@ -881,60 +825,59 @@ bucket_definitions:
881
825
  );
882
826
  const bucketStorage = factory.getInstance(syncRules);
883
827
 
828
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
884
829
  const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
885
830
 
886
831
  // Pre-setup
887
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
888
- await batch.markAllSnapshotDone('1/1');
889
- await batch.save({
890
- sourceTable,
891
- tag: storage.SaveOperationTag.INSERT,
892
- after: {
893
- id: 'test1',
894
- description: 'test1a'
895
- },
896
- afterReplicaId: rid2('test1', 'test1a')
897
- });
832
+ await writer.markAllSnapshotDone('1/1');
833
+ await writer.save({
834
+ sourceTable,
835
+ tag: storage.SaveOperationTag.INSERT,
836
+ after: {
837
+ id: 'test1',
838
+ description: 'test1a'
839
+ },
840
+ afterReplicaId: rid2('test1', 'test1a')
898
841
  });
842
+ const result1 = await writer.flush();
899
843
 
900
844
  const checkpoint1 = result1?.flushed_op ?? 0n;
901
845
 
902
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
903
- // Unchanged, but has a before id
904
- await batch.save({
905
- sourceTable,
906
- tag: storage.SaveOperationTag.UPDATE,
907
- before: {
908
- id: 'test1',
909
- description: 'test1a'
910
- },
911
- beforeReplicaId: rid2('test1', 'test1a'),
912
- after: {
913
- id: 'test1',
914
- description: 'test1b'
915
- },
916
- afterReplicaId: rid2('test1', 'test1b')
917
- });
846
+ // Unchanged, but has a before id
847
+ await writer.save({
848
+ sourceTable,
849
+ tag: storage.SaveOperationTag.UPDATE,
850
+ before: {
851
+ id: 'test1',
852
+ description: 'test1a'
853
+ },
854
+ beforeReplicaId: rid2('test1', 'test1a'),
855
+ after: {
856
+ id: 'test1',
857
+ description: 'test1b'
858
+ },
859
+ afterReplicaId: rid2('test1', 'test1b')
918
860
  });
919
-
920
- const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
921
- // Delete
922
- await batch.save({
923
- sourceTable,
924
- tag: storage.SaveOperationTag.DELETE,
925
- before: {
926
- id: 'test1',
927
- description: 'test1b'
928
- },
929
- beforeReplicaId: rid2('test1', 'test1b'),
930
- after: undefined
931
- });
861
+ const result2 = await writer.flush();
862
+
863
+ // Delete
864
+ await writer.save({
865
+ sourceTable,
866
+ tag: storage.SaveOperationTag.DELETE,
867
+ before: {
868
+ id: 'test1',
869
+ description: 'test1b'
870
+ },
871
+ beforeReplicaId: rid2('test1', 'test1b'),
872
+ after: undefined
932
873
  });
874
+ const result3 = await writer.flush();
933
875
 
934
876
  const checkpoint3 = result3!.flushed_op;
935
877
 
878
+ const request = bucketRequest(syncRules, 'global[]');
936
879
  const batch = await test_utils.fromAsync(
937
- bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]]))
880
+ bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }])
938
881
  );
939
882
  const data = batch[0].chunkData.data.map((d) => {
940
883
  return {
@@ -995,60 +938,59 @@ bucket_definitions:
995
938
  );
996
939
  const bucketStorage = factory.getInstance(syncRules);
997
940
 
941
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
998
942
  const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
999
943
 
1000
944
  // Pre-setup
1001
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1002
- await batch.markAllSnapshotDone('1/1');
1003
- await batch.save({
1004
- sourceTable,
1005
- tag: storage.SaveOperationTag.INSERT,
1006
- after: {
1007
- id: 'test1',
1008
- description: 'test1a'
1009
- },
1010
- afterReplicaId: rid2('test1', 'test1a')
1011
- });
945
+ await writer.markAllSnapshotDone('1/1');
946
+ await writer.save({
947
+ sourceTable,
948
+ tag: storage.SaveOperationTag.INSERT,
949
+ after: {
950
+ id: 'test1',
951
+ description: 'test1a'
952
+ },
953
+ afterReplicaId: rid2('test1', 'test1a')
1012
954
  });
955
+ const result1 = await writer.flush();
1013
956
 
1014
957
  const checkpoint1 = result1?.flushed_op ?? 0n;
1015
958
 
1016
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1017
- // Unchanged, but has a before id
1018
- await batch.save({
1019
- sourceTable,
1020
- tag: storage.SaveOperationTag.UPDATE,
1021
- before: {
1022
- id: 'test1',
1023
- description: 'test1a'
1024
- },
1025
- beforeReplicaId: rid2('test1', 'test1a'),
1026
- after: {
1027
- id: 'test1',
1028
- description: 'test1a'
1029
- },
1030
- afterReplicaId: rid2('test1', 'test1a')
1031
- });
959
+ // Unchanged, but has a before id
960
+ await writer.save({
961
+ sourceTable,
962
+ tag: storage.SaveOperationTag.UPDATE,
963
+ before: {
964
+ id: 'test1',
965
+ description: 'test1a'
966
+ },
967
+ beforeReplicaId: rid2('test1', 'test1a'),
968
+ after: {
969
+ id: 'test1',
970
+ description: 'test1a'
971
+ },
972
+ afterReplicaId: rid2('test1', 'test1a')
1032
973
  });
1033
-
1034
- const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1035
- // Delete
1036
- await batch.save({
1037
- sourceTable,
1038
- tag: storage.SaveOperationTag.DELETE,
1039
- before: {
1040
- id: 'test1',
1041
- description: 'test1a'
1042
- },
1043
- beforeReplicaId: rid2('test1', 'test1a'),
1044
- after: undefined
1045
- });
974
+ const result2 = await writer.flush();
975
+
976
+ // Delete
977
+ await writer.save({
978
+ sourceTable,
979
+ tag: storage.SaveOperationTag.DELETE,
980
+ before: {
981
+ id: 'test1',
982
+ description: 'test1a'
983
+ },
984
+ beforeReplicaId: rid2('test1', 'test1a'),
985
+ after: undefined
1046
986
  });
987
+ const result3 = await writer.flush();
1047
988
 
1048
989
  const checkpoint3 = result3!.flushed_op;
1049
990
 
991
+ const request = bucketRequest(syncRules, 'global[]');
1050
992
  const batch = await test_utils.fromAsync(
1051
- bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]]))
993
+ bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }])
1052
994
  );
1053
995
  const data = batch[0].chunkData.data.map((d) => {
1054
996
  return {
@@ -1098,66 +1040,63 @@ bucket_definitions:
1098
1040
  )
1099
1041
  );
1100
1042
  const bucketStorage = factory.getInstance(syncRules);
1043
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1044
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1045
+ await writer.markAllSnapshotDone('1/1');
1046
+
1047
+ const largeDescription = '0123456789'.repeat(12_000_00);
1048
+
1049
+ await writer.save({
1050
+ sourceTable,
1051
+ tag: storage.SaveOperationTag.INSERT,
1052
+ after: {
1053
+ id: 'test1',
1054
+ description: 'test1'
1055
+ },
1056
+ afterReplicaId: test_utils.rid('test1')
1057
+ });
1101
1058
 
1102
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1103
- await batch.markAllSnapshotDone('1/1');
1104
- const sourceTable = TEST_TABLE;
1105
-
1106
- const largeDescription = '0123456789'.repeat(12_000_00);
1107
-
1108
- await batch.save({
1109
- sourceTable,
1110
- tag: storage.SaveOperationTag.INSERT,
1111
- after: {
1112
- id: 'test1',
1113
- description: 'test1'
1114
- },
1115
- afterReplicaId: test_utils.rid('test1')
1116
- });
1117
-
1118
- await batch.save({
1119
- sourceTable,
1120
- tag: storage.SaveOperationTag.INSERT,
1121
- after: {
1122
- id: 'large1',
1123
- description: largeDescription
1124
- },
1125
- afterReplicaId: test_utils.rid('large1')
1126
- });
1127
-
1128
- // Large enough to split the returned batch
1129
- await batch.save({
1130
- sourceTable,
1131
- tag: storage.SaveOperationTag.INSERT,
1132
- after: {
1133
- id: 'large2',
1134
- description: largeDescription
1135
- },
1136
- afterReplicaId: test_utils.rid('large2')
1137
- });
1059
+ await writer.save({
1060
+ sourceTable,
1061
+ tag: storage.SaveOperationTag.INSERT,
1062
+ after: {
1063
+ id: 'large1',
1064
+ description: largeDescription
1065
+ },
1066
+ afterReplicaId: test_utils.rid('large1')
1067
+ });
1138
1068
 
1139
- await batch.save({
1140
- sourceTable,
1141
- tag: storage.SaveOperationTag.INSERT,
1142
- after: {
1143
- id: 'test3',
1144
- description: 'test3'
1145
- },
1146
- afterReplicaId: test_utils.rid('test3')
1147
- });
1069
+ // Large enough to split the returned batch
1070
+ await writer.save({
1071
+ sourceTable,
1072
+ tag: storage.SaveOperationTag.INSERT,
1073
+ after: {
1074
+ id: 'large2',
1075
+ description: largeDescription
1076
+ },
1077
+ afterReplicaId: test_utils.rid('large2')
1078
+ });
1148
1079
 
1149
- await batch.commit('1/1');
1080
+ await writer.save({
1081
+ sourceTable,
1082
+ tag: storage.SaveOperationTag.INSERT,
1083
+ after: {
1084
+ id: 'test3',
1085
+ description: 'test3'
1086
+ },
1087
+ afterReplicaId: test_utils.rid('test3')
1150
1088
  });
1151
1089
 
1090
+ await writer.commit('1/1');
1091
+
1152
1092
  const { checkpoint } = await bucketStorage.getCheckpoint();
1153
1093
 
1154
1094
  const options: storage.BucketDataBatchOptions = {
1155
1095
  chunkLimitBytes: 16 * 1024 * 1024
1156
1096
  };
1157
1097
 
1158
- const batch1 = await test_utils.fromAsync(
1159
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
1160
- );
1098
+ const request = bucketRequest(syncRules, 'global[]');
1099
+ const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
1161
1100
  expect(test_utils.getBatchData(batch1)).toEqual([
1162
1101
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1163
1102
  { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
@@ -1171,7 +1110,7 @@ bucket_definitions:
1171
1110
  const batch2 = await test_utils.fromAsync(
1172
1111
  bucketStorage.getBucketDataBatch(
1173
1112
  checkpoint,
1174
- bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
1113
+ [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }],
1175
1114
  options
1176
1115
  )
1177
1116
  );
@@ -1188,7 +1127,7 @@ bucket_definitions:
1188
1127
  const batch3 = await test_utils.fromAsync(
1189
1128
  bucketStorage.getBucketDataBatch(
1190
1129
  checkpoint,
1191
- bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
1130
+ [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }],
1192
1131
  options
1193
1132
  )
1194
1133
  );
@@ -1213,31 +1152,28 @@ bucket_definitions:
1213
1152
  )
1214
1153
  );
1215
1154
  const bucketStorage = factory.getInstance(syncRules);
1155
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1156
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1157
+ await writer.markAllSnapshotDone('1/1');
1216
1158
 
1217
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1218
- await batch.markAllSnapshotDone('1/1');
1219
- const sourceTable = TEST_TABLE;
1220
-
1221
- for (let i = 1; i <= 6; i++) {
1222
- await batch.save({
1223
- sourceTable,
1224
- tag: storage.SaveOperationTag.INSERT,
1225
- after: {
1226
- id: `test${i}`,
1227
- description: `test${i}`
1228
- },
1229
- afterReplicaId: `test${i}`
1230
- });
1231
- }
1159
+ for (let i = 1; i <= 6; i++) {
1160
+ await writer.save({
1161
+ sourceTable,
1162
+ tag: storage.SaveOperationTag.INSERT,
1163
+ after: {
1164
+ id: `test${i}`,
1165
+ description: `test${i}`
1166
+ },
1167
+ afterReplicaId: `test${i}`
1168
+ });
1169
+ }
1232
1170
 
1233
- await batch.commit('1/1');
1234
- });
1171
+ await writer.commit('1/1');
1235
1172
 
1236
1173
  const { checkpoint } = await bucketStorage.getCheckpoint();
1237
1174
 
1238
- const batch1 = await test_utils.oneFromAsync(
1239
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), { limit: 4 })
1240
- );
1175
+ const request = bucketRequest(syncRules, 'global[]');
1176
+ const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
1241
1177
 
1242
1178
  expect(test_utils.getBatchData(batch1)).toEqual([
1243
1179
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -1253,13 +1189,9 @@ bucket_definitions:
1253
1189
  });
1254
1190
 
1255
1191
  const batch2 = await test_utils.oneFromAsync(
1256
- bucketStorage.getBucketDataBatch(
1257
- checkpoint,
1258
- bucketRequestMap(syncRules, [['global[]', BigInt(batch1.chunkData.next_after)]]),
1259
- {
1260
- limit: 4
1261
- }
1262
- )
1192
+ bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
1193
+ limit: 4
1194
+ })
1263
1195
  );
1264
1196
  expect(test_utils.getBatchData(batch2)).toEqual([
1265
1197
  { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
@@ -1273,13 +1205,9 @@ bucket_definitions:
1273
1205
  });
1274
1206
 
1275
1207
  const batch3 = await test_utils.fromAsync(
1276
- bucketStorage.getBucketDataBatch(
1277
- checkpoint,
1278
- bucketRequestMap(syncRules, [['global[]', BigInt(batch2.chunkData.next_after)]]),
1279
- {
1280
- limit: 4
1281
- }
1282
- )
1208
+ bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
1209
+ limit: 4
1210
+ })
1283
1211
  );
1284
1212
  expect(test_utils.getBatchData(batch3)).toEqual([]);
1285
1213
 
@@ -1304,48 +1232,41 @@ bucket_definitions:
1304
1232
  )
1305
1233
  );
1306
1234
  const bucketStorage = factory.getInstance(syncRules);
1235
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1236
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1237
+ await writer.markAllSnapshotDone('1/1');
1307
1238
 
1308
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1309
- await batch.markAllSnapshotDone('1/1');
1310
- const sourceTable = TEST_TABLE;
1311
-
1312
- for (let i = 1; i <= 10; i++) {
1313
- await batch.save({
1314
- sourceTable,
1315
- tag: storage.SaveOperationTag.INSERT,
1316
- after: {
1317
- id: `test${i}`,
1318
- description: `test${i}`,
1319
- bucket: i == 1 ? 'global1' : 'global2'
1320
- },
1321
- afterReplicaId: `test${i}`
1322
- });
1323
- }
1239
+ for (let i = 1; i <= 10; i++) {
1240
+ await writer.save({
1241
+ sourceTable,
1242
+ tag: storage.SaveOperationTag.INSERT,
1243
+ after: {
1244
+ id: `test${i}`,
1245
+ description: `test${i}`,
1246
+ bucket: i == 1 ? 'global1' : 'global2'
1247
+ },
1248
+ afterReplicaId: `test${i}`
1249
+ });
1250
+ }
1324
1251
 
1325
- await batch.commit('1/1');
1326
- });
1252
+ await writer.commit('1/1');
1327
1253
 
1328
1254
  const { checkpoint } = await bucketStorage.getCheckpoint();
1255
+ const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
1256
+ const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
1329
1257
  const batch = await test_utils.fromAsync(
1330
- bucketStorage.getBucketDataBatch(
1331
- checkpoint,
1332
- bucketRequestMap(syncRules, [
1333
- ['global1[]', 0n],
1334
- ['global2[]', 0n]
1335
- ]),
1336
- options
1337
- )
1258
+ bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options)
1338
1259
  );
1339
1260
 
1340
- return { syncRules, batch };
1261
+ return { batch, global1Request, global2Request };
1341
1262
  };
1342
1263
 
1343
1264
  test('batch has_more (1)', async () => {
1344
- const { batch, syncRules } = await setup({ limit: 5 });
1265
+ const { batch, global1Request, global2Request } = await setup({ limit: 5 });
1345
1266
  expect(batch.length).toEqual(2);
1346
1267
 
1347
- expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
1348
- expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1268
+ expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1269
+ expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1349
1270
 
1350
1271
  expect(test_utils.getBatchData(batch[0])).toEqual([
1351
1272
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -1372,11 +1293,11 @@ bucket_definitions:
1372
1293
  });
1373
1294
 
1374
1295
  test('batch has_more (2)', async () => {
1375
- const { batch, syncRules } = await setup({ limit: 11 });
1296
+ const { batch, global1Request, global2Request } = await setup({ limit: 11 });
1376
1297
  expect(batch.length).toEqual(2);
1377
1298
 
1378
- expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
1379
- expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1299
+ expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1300
+ expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1380
1301
 
1381
1302
  expect(test_utils.getBatchData(batch[0])).toEqual([
1382
1303
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -1409,12 +1330,12 @@ bucket_definitions:
1409
1330
 
1410
1331
  test('batch has_more (3)', async () => {
1411
1332
  // 50 bytes is more than 1 row, less than 2 rows
1412
- const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 });
1333
+ const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
1413
1334
 
1414
1335
  expect(batch.length).toEqual(3);
1415
- expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
1416
- expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1417
- expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1336
+ expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1337
+ expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1338
+ expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
1418
1339
 
1419
1340
  expect(test_utils.getBatchData(batch[0])).toEqual([
1420
1341
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
@@ -1458,10 +1379,9 @@ bucket_definitions:
1458
1379
 
1459
1380
  const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
1460
1381
  const storage = f.getInstance(r.persisted_sync_rules!);
1461
- await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1462
- await batch.markAllSnapshotDone('1/0');
1463
- await batch.keepalive('1/0');
1464
- });
1382
+ await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
1383
+ await writer.markAllSnapshotDone('1/0');
1384
+ await writer.keepalive('1/0');
1465
1385
 
1466
1386
  await f.getStorageMetrics();
1467
1387
  // We don't care about the specific values here
@@ -1487,36 +1407,35 @@ bucket_definitions:
1487
1407
  )
1488
1408
  );
1489
1409
  const bucketStorage = factory.getInstance(syncRules);
1490
-
1491
- const sourceTable = test_utils.makeTestTable('test', ['id'], config);
1492
- const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id'], config);
1493
-
1494
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1495
- await batch.markAllSnapshotDone('1/1');
1496
- // This saves a record to current_data, but not bucket_data.
1497
- // This causes a checkpoint to be created without increasing the op_id sequence.
1498
- await batch.save({
1499
- sourceTable: sourceTableIgnore,
1500
- tag: storage.SaveOperationTag.INSERT,
1501
- after: {
1502
- id: 'test1'
1503
- },
1504
- afterReplicaId: test_utils.rid('test1')
1505
- });
1410
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1411
+
1412
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
1413
+ const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
1414
+
1415
+ await writer.markAllSnapshotDone('1/1');
1416
+ // This saves a record to current_data, but not bucket_data.
1417
+ // This causes a checkpoint to be created without increasing the op_id sequence.
1418
+ await writer.save({
1419
+ sourceTable: sourceTableIgnore,
1420
+ tag: storage.SaveOperationTag.INSERT,
1421
+ after: {
1422
+ id: 'test1'
1423
+ },
1424
+ afterReplicaId: test_utils.rid('test1')
1506
1425
  });
1426
+ const result1 = await writer.flush();
1507
1427
 
1508
1428
  const checkpoint1 = result1!.flushed_op;
1509
1429
 
1510
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1511
- await batch.save({
1512
- sourceTable: sourceTable,
1513
- tag: storage.SaveOperationTag.INSERT,
1514
- after: {
1515
- id: 'test2'
1516
- },
1517
- afterReplicaId: test_utils.rid('test2')
1518
- });
1430
+ await writer.save({
1431
+ sourceTable: sourceTable,
1432
+ tag: storage.SaveOperationTag.INSERT,
1433
+ after: {
1434
+ id: 'test2'
1435
+ },
1436
+ afterReplicaId: test_utils.rid('test2')
1519
1437
  });
1438
+ const result2 = await writer.flush();
1520
1439
 
1521
1440
  const checkpoint2 = result2!.flushed_op;
1522
1441
  // we expect 0n and 1n, or 1n and 2n.
@@ -1539,35 +1458,27 @@ bucket_definitions:
1539
1458
  )
1540
1459
  );
1541
1460
  const bucketStorage = factory.getInstance(syncRules);
1542
-
1543
- const sourceTable = TEST_TABLE;
1544
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1545
- await batch.markAllSnapshotDone('1/1');
1546
- await batch.save({
1547
- sourceTable,
1548
- tag: storage.SaveOperationTag.INSERT,
1549
- after: {
1550
- id: 'test1',
1551
- description: 'test1a'
1552
- },
1553
- afterReplicaId: test_utils.rid('test1')
1554
- });
1555
- await batch.commit('1/1');
1461
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1462
+
1463
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1464
+ await writer.markAllSnapshotDone('1/1');
1465
+ await writer.save({
1466
+ sourceTable,
1467
+ tag: storage.SaveOperationTag.INSERT,
1468
+ after: {
1469
+ id: 'test1',
1470
+ description: 'test1a'
1471
+ },
1472
+ afterReplicaId: test_utils.rid('test1')
1556
1473
  });
1474
+ await writer.commit('1/1');
1557
1475
  const { checkpoint } = await bucketStorage.getCheckpoint();
1558
1476
 
1559
- const checksums = [
1560
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
1561
- ];
1562
- expect(checksums).toEqual([
1563
- { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
1564
- ]);
1565
- const checksums2 = [
1566
- ...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
1567
- ];
1568
- expect(checksums2).toEqual([
1569
- { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
1570
- ]);
1477
+ const request = bucketRequest(syncRules, 'global[]');
1478
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
1479
+ expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1480
+ const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
1481
+ expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1571
1482
  });
1572
1483
 
1573
1484
  testChecksumBatching(config);
@@ -1586,28 +1497,26 @@ bucket_definitions:
1586
1497
  )
1587
1498
  );
1588
1499
  const bucketStorage = factory.getInstance(syncRules);
1500
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1501
+ await writer.markAllSnapshotDone('1/1');
1502
+ await writer.commit('1/1');
1589
1503
 
1590
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1591
- await batch.markAllSnapshotDone('1/1');
1592
- await batch.commit('1/1');
1593
-
1594
- const cp1 = await bucketStorage.getCheckpoint();
1595
- expect(cp1.lsn).toEqual('1/1');
1504
+ const cp1 = await bucketStorage.getCheckpoint();
1505
+ expect(cp1.lsn).toEqual('1/1');
1596
1506
 
1597
- await batch.commit('2/1', { createEmptyCheckpoints: true });
1598
- const cp2 = await bucketStorage.getCheckpoint();
1599
- expect(cp2.lsn).toEqual('2/1');
1507
+ await writer.commit('2/1', { createEmptyCheckpoints: true });
1508
+ const cp2 = await bucketStorage.getCheckpoint();
1509
+ expect(cp2.lsn).toEqual('2/1');
1600
1510
 
1601
- await batch.keepalive('3/1');
1602
- const cp3 = await bucketStorage.getCheckpoint();
1603
- expect(cp3.lsn).toEqual('3/1');
1511
+ await writer.keepalive('3/1');
1512
+ const cp3 = await bucketStorage.getCheckpoint();
1513
+ expect(cp3.lsn).toEqual('3/1');
1604
1514
 
1605
- // For the last one, we skip creating empty checkpoints
1606
- // This means the LSN stays at 3/1.
1607
- await batch.commit('4/1', { createEmptyCheckpoints: false });
1608
- const cp4 = await bucketStorage.getCheckpoint();
1609
- expect(cp4.lsn).toEqual('3/1');
1610
- });
1515
+ // For the last one, we skip creating empty checkpoints
1516
+ // This means the LSN stays at 3/1.
1517
+ await writer.commit('4/1', { createEmptyCheckpoints: false });
1518
+ const cp4 = await bucketStorage.getCheckpoint();
1519
+ expect(cp4.lsn).toEqual('3/1');
1611
1520
  });
1612
1521
 
1613
1522
  test('empty checkpoints (2)', async () => {
@@ -1626,40 +1535,38 @@ bucket_definitions:
1626
1535
  )
1627
1536
  );
1628
1537
  const bucketStorage = factory.getInstance(syncRules);
1538
+ await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1539
+ await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1540
+ const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
1629
1541
 
1630
- const sourceTable = TEST_TABLE;
1631
- // We simulate two concurrent batches, but nesting is the easiest way to do this.
1632
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch1) => {
1633
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch2) => {
1634
- await batch1.markAllSnapshotDone('1/1');
1635
- await batch1.commit('1/1');
1636
-
1637
- await batch1.commit('2/1', { createEmptyCheckpoints: false });
1638
- const cp2 = await bucketStorage.getCheckpoint();
1639
- expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
1542
+ // We simulate two concurrent batches, but sequential calls are enough for this test.
1543
+ await writer1.markAllSnapshotDone('1/1');
1544
+ await writer1.commit('1/1');
1640
1545
 
1641
- await batch2.save({
1642
- sourceTable,
1643
- tag: storage.SaveOperationTag.INSERT,
1644
- after: {
1645
- id: 'test1',
1646
- description: 'test1a'
1647
- },
1648
- afterReplicaId: test_utils.rid('test1')
1649
- });
1650
- // This simulates what happens on a snapshot processor.
1651
- // This may later change to a flush() rather than commit().
1652
- await batch2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
1546
+ await writer1.commit('2/1', { createEmptyCheckpoints: false });
1547
+ const cp2 = await bucketStorage.getCheckpoint();
1548
+ expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
1549
+
1550
+ await writer2.save({
1551
+ sourceTable,
1552
+ tag: storage.SaveOperationTag.INSERT,
1553
+ after: {
1554
+ id: 'test1',
1555
+ description: 'test1a'
1556
+ },
1557
+ afterReplicaId: test_utils.rid('test1')
1558
+ });
1559
+ // This simulates what happens on a snapshot processor.
1560
+ // This may later change to a flush() rather than commit().
1561
+ await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
1653
1562
 
1654
- const cp3 = await bucketStorage.getCheckpoint();
1655
- expect(cp3.lsn).toEqual('1/1'); // Still unchanged
1563
+ const cp3 = await bucketStorage.getCheckpoint();
1564
+ expect(cp3.lsn).toEqual('1/1'); // Still unchanged
1656
1565
 
1657
- // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
1658
- await batch1.commit('4/1', { createEmptyCheckpoints: false });
1659
- const cp4 = await bucketStorage.getCheckpoint();
1660
- expect(cp4.lsn).toEqual('4/1');
1661
- });
1662
- });
1566
+ // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
1567
+ await writer1.commit('4/1', { createEmptyCheckpoints: false });
1568
+ const cp4 = await bucketStorage.getCheckpoint();
1569
+ expect(cp4.lsn).toEqual('4/1');
1663
1570
  });
1664
1571
 
1665
1572
  test('empty checkpoints (sync rule activation)', async () => {
@@ -1679,29 +1586,24 @@ bucket_definitions:
1679
1586
  );
1680
1587
  const bucketStorage = factory.getInstance(syncRules);
1681
1588
 
1682
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1683
- const result = await batch.commit('1/1', { createEmptyCheckpoints: false });
1684
- expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1685
- // Snapshot is only valid once we reach 3/1
1686
- await batch.markAllSnapshotDone('3/1');
1687
- });
1589
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1590
+ const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
1591
+ expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1592
+ // Snapshot is only valid once we reach 3/1
1593
+ await writer.markAllSnapshotDone('3/1');
1688
1594
 
1689
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1690
- // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
1691
- const result = await batch.commit('2/1', { createEmptyCheckpoints: false });
1692
- expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1693
- });
1595
+ // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
1596
+ const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
1597
+ expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1694
1598
 
1695
1599
  // No empty checkpoint should be created by the commit above.
1696
1600
  const cp1 = await bucketStorage.getCheckpoint();
1697
1601
  expect(cp1.lsn).toEqual(null);
1698
1602
 
1699
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1700
- // After this commit, the snapshot should be valid.
1701
- // We specifically check that this is done even if createEmptyCheckpoints: false.
1702
- const result = await batch.commit('3/1', { createEmptyCheckpoints: false });
1703
- expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true });
1704
- });
1603
+ // After this commit, the snapshot should be valid.
1604
+ // We specifically check that this is done even if createEmptyCheckpoints: false.
1605
+ const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
1606
+ expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
1705
1607
 
1706
1608
  // Now, the checkpoint should advance the sync rules active.
1707
1609
  const cp2 = await bucketStorage.getCheckpoint();
@@ -1710,11 +1612,9 @@ bucket_definitions:
1710
1612
  const activeSyncRules = await factory.getActiveSyncRulesContent();
1711
1613
  expect(activeSyncRules?.id).toEqual(syncRules.id);
1712
1614
 
1713
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1714
- // At this point, it should be a truely empty checkpoint
1715
- const result = await batch.commit('4/1', { createEmptyCheckpoints: false });
1716
- expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false });
1717
- });
1615
+ // At this point, it should be a truely empty checkpoint
1616
+ const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
1617
+ expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
1718
1618
 
1719
1619
  // Unchanged
1720
1620
  const cp3 = await bucketStorage.getCheckpoint();
@@ -1737,44 +1637,46 @@ bucket_definitions:
1737
1637
  )
1738
1638
  );
1739
1639
  const bucketStorage = factory.getInstance(syncRules);
1640
+ await using snapshotWriter = await bucketStorage.createWriter({
1641
+ ...test_utils.BATCH_OPTIONS,
1642
+ skipExistingRows: true
1643
+ });
1644
+ await using streamingWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1645
+ const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
1646
+ const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
1740
1647
 
1741
- const sourceTable = TEST_TABLE;
1742
- // We simulate two concurrent batches, and nesting is the easiest way to do this.
1648
+ // We simulate two concurrent batches; separate writers are enough for this test.
1743
1649
  // For this test, we assume that we start with a row "test1", which is picked up by a snapshot
1744
1650
  // query, right before the delete is streamed. But the snapshot query is only persisted _after_
1745
1651
  // the delete is streamed, and we need to ensure that the streamed delete takes precedence.
1746
- await bucketStorage.startBatch({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true }, async (snapshotBatch) => {
1747
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (streamingBatch) => {
1748
- streamingBatch.save({
1749
- sourceTable,
1750
- tag: storage.SaveOperationTag.DELETE,
1751
- before: {
1752
- id: 'test1'
1753
- },
1754
- beforeReplicaId: test_utils.rid('test1')
1755
- });
1756
- await streamingBatch.commit('2/1');
1757
-
1758
- await snapshotBatch.save({
1759
- sourceTable,
1760
- tag: storage.SaveOperationTag.INSERT,
1761
- after: {
1762
- id: 'test1',
1763
- description: 'test1a'
1764
- },
1765
- afterReplicaId: test_utils.rid('test1')
1766
- });
1767
- await snapshotBatch.markAllSnapshotDone('3/1');
1768
- await snapshotBatch.commit('1/1');
1769
-
1770
- await streamingBatch.keepalive('3/1');
1771
- });
1652
+ await streamingWriter.save({
1653
+ sourceTable: streamingTable,
1654
+ tag: storage.SaveOperationTag.DELETE,
1655
+ before: {
1656
+ id: 'test1'
1657
+ },
1658
+ beforeReplicaId: test_utils.rid('test1')
1772
1659
  });
1660
+ await streamingWriter.commit('2/1');
1661
+
1662
+ await snapshotWriter.save({
1663
+ sourceTable: snapshotTable,
1664
+ tag: storage.SaveOperationTag.INSERT,
1665
+ after: {
1666
+ id: 'test1',
1667
+ description: 'test1a'
1668
+ },
1669
+ afterReplicaId: test_utils.rid('test1')
1670
+ });
1671
+ await snapshotWriter.markAllSnapshotDone('3/1');
1672
+ await snapshotWriter.commit('1/1');
1673
+
1674
+ await streamingWriter.keepalive('3/1');
1773
1675
 
1774
1676
  const cp = await bucketStorage.getCheckpoint();
1775
1677
  expect(cp.lsn).toEqual('3/1');
1776
1678
  const data = await test_utils.fromAsync(
1777
- bucketStorage.getBucketDataBatch(cp.checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
1679
+ bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')])
1778
1680
  );
1779
1681
 
1780
1682
  expect(data).toEqual([]);
@@ -1805,38 +1707,39 @@ bucket_definitions:
1805
1707
  )
1806
1708
  );
1807
1709
  const bucketStorage = factory.getInstance(syncRules);
1808
-
1809
- const sourceTable = test_utils.makeTestTable('test', ['id'], config);
1810
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1811
- await batch.markAllSnapshotDone('1/1');
1812
- for (let u of ['u1', 'u2', 'u3', 'u4']) {
1813
- for (let t of ['t1', 't2', 't3', 't4']) {
1814
- const id = `${t}_${u}`;
1815
- await batch.save({
1816
- sourceTable,
1817
- tag: storage.SaveOperationTag.INSERT,
1818
- after: {
1819
- id,
1820
- description: `${t} description`,
1821
- user_id: u
1822
- },
1823
- afterReplicaId: test_utils.rid(id)
1824
- });
1825
- }
1710
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1711
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1712
+ await writer.markAllSnapshotDone('1/1');
1713
+ for (let u of ['u1', 'u2', 'u3', 'u4']) {
1714
+ for (let t of ['t1', 't2', 't3', 't4']) {
1715
+ const id = `${t}_${u}`;
1716
+ await writer.save({
1717
+ sourceTable,
1718
+ tag: storage.SaveOperationTag.INSERT,
1719
+ after: {
1720
+ id,
1721
+ description: `${t} description`,
1722
+ user_id: u
1723
+ },
1724
+ afterReplicaId: test_utils.rid(id)
1725
+ });
1826
1726
  }
1827
- await batch.commit('1/1');
1828
- });
1727
+ }
1728
+ await writer.commit('1/1');
1829
1729
  const { checkpoint } = await bucketStorage.getCheckpoint();
1830
1730
 
1831
1731
  bucketStorage.clearChecksumCache();
1832
- const buckets = bucketRequests(syncRules, ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]']);
1833
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()];
1732
+ const users = ['u1', 'u2', 'u3', 'u4'];
1733
+ const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
1734
+ const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
1735
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
1834
1736
  checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
1835
- expect(checksums).toEqual([
1836
- { bucket: bucketRequest(syncRules, 'user["u1"]').bucket, count: 4, checksum: 346204588 },
1837
- { bucket: bucketRequest(syncRules, 'user["u2"]').bucket, count: 4, checksum: 5261081 },
1838
- { bucket: bucketRequest(syncRules, 'user["u3"]').bucket, count: 4, checksum: 134760718 },
1839
- { bucket: bucketRequest(syncRules, 'user["u4"]').bucket, count: 4, checksum: -302639724 }
1840
- ]);
1737
+ const expected = bucketRequests.map((request, index) => ({
1738
+ bucket: request.bucket,
1739
+ count: 4,
1740
+ checksum: expectedChecksums[index]
1741
+ }));
1742
+ expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
1743
+ expect(checksums).toEqual(expected);
1841
1744
  });
1842
1745
  }