@powersync/service-core-tests 0.15.0 → 0.15.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/dist/test-utils/general-utils.d.ts +13 -1
  3. package/dist/test-utils/general-utils.js +30 -1
  4. package/dist/test-utils/general-utils.js.map +1 -1
  5. package/dist/test-utils/stream_utils.js +2 -2
  6. package/dist/test-utils/stream_utils.js.map +1 -1
  7. package/dist/tests/register-compacting-tests.js +266 -257
  8. package/dist/tests/register-compacting-tests.js.map +1 -1
  9. package/dist/tests/register-data-storage-checkpoint-tests.js +36 -57
  10. package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
  11. package/dist/tests/register-data-storage-data-tests.js +839 -863
  12. package/dist/tests/register-data-storage-data-tests.js.map +1 -1
  13. package/dist/tests/register-data-storage-parameter-tests.js +228 -236
  14. package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
  15. package/dist/tests/register-parameter-compacting-tests.js +81 -89
  16. package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
  17. package/dist/tests/register-sync-tests.js +468 -462
  18. package/dist/tests/register-sync-tests.js.map +1 -1
  19. package/package.json +3 -3
  20. package/src/test-utils/general-utils.ts +41 -2
  21. package/src/test-utils/stream_utils.ts +2 -2
  22. package/src/tests/register-compacting-tests.ts +279 -270
  23. package/src/tests/register-data-storage-checkpoint-tests.ts +36 -57
  24. package/src/tests/register-data-storage-data-tests.ts +673 -770
  25. package/src/tests/register-data-storage-parameter-tests.ts +245 -257
  26. package/src/tests/register-parameter-compacting-tests.ts +84 -92
  27. package/src/tests/register-sync-tests.ts +375 -391
  28. package/tsconfig.tsbuildinfo +1 -1
@@ -54,7 +54,6 @@ import { CURRENT_STORAGE_VERSION, getUuidReplicaIdentityBson, reduceBucket, stor
54
54
  import { describe, expect, test } from 'vitest';
55
55
  import * as test_utils from '../test-utils/test-utils-index.js';
56
56
  import { bucketRequest } from '../test-utils/test-utils-index.js';
57
- import { bucketRequestMap, bucketRequests } from './util.js';
58
57
  /**
59
58
  * Normalize data from OplogEntries for comparison in tests.
60
59
  * Tests typically expect the stringified result
@@ -78,7 +77,6 @@ const normalizeOplogData = (data) => {
78
77
  export function registerDataStorageDataTests(config) {
79
78
  const generateStorageFactory = config.factory;
80
79
  const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
81
- const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
82
80
  test('removing row', async () => {
83
81
  const env_1 = { stack: [], error: void 0, hasError: false };
84
82
  try {
@@ -90,27 +88,27 @@ bucket_definitions:
90
88
  - SELECT id, description FROM "%"
91
89
  `, { storageVersion }));
92
90
  const bucketStorage = factory.getInstance(syncRules);
93
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
94
- const sourceTable = TEST_TABLE;
95
- await batch.markAllSnapshotDone('1/1');
96
- await batch.save({
97
- sourceTable,
98
- tag: storage.SaveOperationTag.INSERT,
99
- after: {
100
- id: 'test1',
101
- description: 'test1'
102
- },
103
- afterReplicaId: test_utils.rid('test1')
104
- });
105
- await batch.save({
106
- sourceTable,
107
- tag: storage.SaveOperationTag.DELETE,
108
- beforeReplicaId: test_utils.rid('test1')
109
- });
110
- await batch.commit('1/1');
91
+ const writer = __addDisposableResource(env_1, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
92
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
93
+ await writer.markAllSnapshotDone('1/1');
94
+ await writer.save({
95
+ sourceTable: testTable,
96
+ tag: storage.SaveOperationTag.INSERT,
97
+ after: {
98
+ id: 'test1',
99
+ description: 'test1'
100
+ },
101
+ afterReplicaId: test_utils.rid('test1')
111
102
  });
103
+ await writer.save({
104
+ sourceTable: testTable,
105
+ tag: storage.SaveOperationTag.DELETE,
106
+ beforeReplicaId: test_utils.rid('test1')
107
+ });
108
+ await writer.commit('1/1');
112
109
  const { checkpoint } = await bucketStorage.getCheckpoint();
113
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
110
+ const request = bucketRequest(syncRules, 'global[]');
111
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
114
112
  const data = batch[0].chunkData.data.map((d) => {
115
113
  return {
116
114
  op: d.op,
@@ -124,12 +122,10 @@ bucket_definitions:
124
122
  { op: 'PUT', object_id: 'test1', checksum: c1 },
125
123
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
126
124
  ]);
127
- const checksums = [
128
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
129
- ];
125
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
130
126
  expect(checksums).toEqual([
131
127
  {
132
- bucket: bucketRequest(syncRules, 'global[]').bucket,
128
+ bucket: request.bucket,
133
129
  checksum: (c1 + c2) & 0xffffffff,
134
130
  count: 2
135
131
  }
@@ -156,31 +152,28 @@ bucket_definitions:
156
152
  - SELECT id, description FROM "%"
157
153
  `, { storageVersion }));
158
154
  const bucketStorage = factory.getInstance(syncRules);
159
- const sourceTable = TEST_TABLE;
160
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
161
- await batch.markAllSnapshotDone('1/1');
162
- await batch.save({
163
- sourceTable,
164
- tag: storage.SaveOperationTag.DELETE,
165
- beforeReplicaId: test_utils.rid('test1')
166
- });
167
- await batch.commit('0/1');
155
+ const writer = __addDisposableResource(env_2, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
156
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
157
+ await writer.markAllSnapshotDone('1/1');
158
+ await writer.save({
159
+ sourceTable: testTable,
160
+ tag: storage.SaveOperationTag.DELETE,
161
+ beforeReplicaId: test_utils.rid('test1')
168
162
  });
169
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
170
- const sourceTable = TEST_TABLE;
171
- await batch.save({
172
- sourceTable,
173
- tag: storage.SaveOperationTag.INSERT,
174
- after: {
175
- id: 'test1',
176
- description: 'test1'
177
- },
178
- afterReplicaId: test_utils.rid('test1')
179
- });
180
- await batch.commit('2/1');
163
+ await writer.commit('0/1');
164
+ await writer.save({
165
+ sourceTable: testTable,
166
+ tag: storage.SaveOperationTag.INSERT,
167
+ after: {
168
+ id: 'test1',
169
+ description: 'test1'
170
+ },
171
+ afterReplicaId: test_utils.rid('test1')
181
172
  });
173
+ await writer.commit('2/1');
182
174
  const { checkpoint } = await bucketStorage.getCheckpoint();
183
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
175
+ const request = bucketRequest(syncRules, 'global[]');
176
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
184
177
  const data = batch[0].chunkData.data.map((d) => {
185
178
  return {
186
179
  op: d.op,
@@ -190,12 +183,10 @@ bucket_definitions:
190
183
  });
191
184
  const c1 = 2871785649;
192
185
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
193
- const checksums = [
194
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
195
- ];
186
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
196
187
  expect(checksums).toEqual([
197
188
  {
198
- bucket: bucketRequest(syncRules, 'global[]').bucket,
189
+ bucket: request.bucket,
199
190
  checksum: c1 & 0xffffffff,
200
191
  count: 1
201
192
  }
@@ -223,35 +214,32 @@ bucket_definitions:
223
214
  - SELECT id, description FROM "%"
224
215
  `, { storageVersion }));
225
216
  const bucketStorage = factory.getInstance(syncRules);
226
- const sourceTable = TEST_TABLE;
227
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
228
- await batch.markAllSnapshotDone('1/1');
229
- await batch.save({
230
- sourceTable,
231
- tag: storage.SaveOperationTag.DELETE,
232
- beforeReplicaId: test_utils.rid('test1')
233
- });
234
- await batch.commit('0/1');
217
+ const writer = __addDisposableResource(env_3, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
218
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
219
+ await writer.markAllSnapshotDone('1/1');
220
+ await writer.save({
221
+ sourceTable: testTable,
222
+ tag: storage.SaveOperationTag.DELETE,
223
+ beforeReplicaId: test_utils.rid('test1')
235
224
  });
236
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
237
- const sourceTable = TEST_TABLE;
238
- await batch.save({
239
- sourceTable,
240
- tag: storage.SaveOperationTag.UPDATE,
241
- before: {
242
- id: 'test1'
243
- },
244
- after: {
245
- id: 'test1',
246
- description: 'test1'
247
- },
248
- beforeReplicaId: test_utils.rid('test1'),
249
- afterReplicaId: test_utils.rid('test1')
250
- });
251
- await batch.commit('2/1');
225
+ await writer.commit('0/1');
226
+ await writer.save({
227
+ sourceTable: testTable,
228
+ tag: storage.SaveOperationTag.UPDATE,
229
+ before: {
230
+ id: 'test1'
231
+ },
232
+ after: {
233
+ id: 'test1',
234
+ description: 'test1'
235
+ },
236
+ beforeReplicaId: test_utils.rid('test1'),
237
+ afterReplicaId: test_utils.rid('test1')
252
238
  });
239
+ await writer.commit('2/1');
253
240
  const { checkpoint } = await bucketStorage.getCheckpoint();
254
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
241
+ const request = bucketRequest(syncRules, 'global[]');
242
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
255
243
  const data = batch[0].chunkData.data.map((d) => {
256
244
  return {
257
245
  op: d.op,
@@ -261,12 +249,10 @@ bucket_definitions:
261
249
  });
262
250
  const c1 = 2871785649;
263
251
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
264
- const checksums = [
265
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
266
- ];
252
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
267
253
  expect(checksums).toEqual([
268
254
  {
269
- bucket: bucketRequest(syncRules, 'global[]').bucket,
255
+ bucket: request.bucket,
270
256
  checksum: c1 & 0xffffffff,
271
257
  count: 1
272
258
  }
@@ -295,27 +281,27 @@ bucket_definitions:
295
281
  storageVersion
296
282
  }));
297
283
  const bucketStorage = factory.getInstance(syncRules);
298
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
299
- const sourceTable = TEST_TABLE;
300
- await batch.markAllSnapshotDone('1/1');
301
- await batch.save({
302
- sourceTable,
303
- tag: storage.SaveOperationTag.DELETE,
304
- beforeReplicaId: test_utils.rid('test1')
305
- });
306
- await batch.save({
307
- sourceTable,
308
- tag: storage.SaveOperationTag.INSERT,
309
- after: {
310
- id: 'test1',
311
- description: 'test1'
312
- },
313
- afterReplicaId: test_utils.rid('test1')
314
- });
315
- await batch.commit('1/1');
284
+ const writer = __addDisposableResource(env_4, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
285
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
286
+ await writer.markAllSnapshotDone('1/1');
287
+ await writer.save({
288
+ sourceTable: testTable,
289
+ tag: storage.SaveOperationTag.DELETE,
290
+ beforeReplicaId: test_utils.rid('test1')
291
+ });
292
+ await writer.save({
293
+ sourceTable: testTable,
294
+ tag: storage.SaveOperationTag.INSERT,
295
+ after: {
296
+ id: 'test1',
297
+ description: 'test1'
298
+ },
299
+ afterReplicaId: test_utils.rid('test1')
316
300
  });
301
+ await writer.commit('1/1');
317
302
  const { checkpoint } = await bucketStorage.getCheckpoint();
318
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
303
+ const request = bucketRequest(syncRules, 'global[]');
304
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
319
305
  const data = batch[0].chunkData.data.map((d) => {
320
306
  return {
321
307
  op: d.op,
@@ -325,12 +311,10 @@ bucket_definitions:
325
311
  });
326
312
  const c1 = 2871785649;
327
313
  expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
328
- const checksums = [
329
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
330
- ];
314
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
331
315
  expect(checksums).toEqual([
332
316
  {
333
- bucket: bucketRequest(syncRules, 'global[]').bucket,
317
+ bucket: request.bucket,
334
318
  checksum: c1 & 0xffffffff,
335
319
  count: 1
336
320
  }
@@ -359,46 +343,73 @@ bucket_definitions:
359
343
  storageVersion
360
344
  }));
361
345
  const bucketStorage = factory.getInstance(syncRules);
362
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
363
- const sourceTable = TEST_TABLE;
364
- await batch.markAllSnapshotDone('1/1');
365
- await batch.save({
366
- sourceTable,
367
- tag: storage.SaveOperationTag.INSERT,
368
- after: {
369
- id: 'test1',
370
- description: 'test1'
371
- },
372
- afterReplicaId: test_utils.rid('test1')
373
- });
374
- await batch.save({
375
- sourceTable,
376
- tag: storage.SaveOperationTag.DELETE,
377
- beforeReplicaId: test_utils.rid('test1')
378
- });
379
- await batch.save({
380
- sourceTable,
381
- tag: storage.SaveOperationTag.INSERT,
382
- after: {
383
- id: 'test1',
384
- description: 'test1'
385
- },
386
- afterReplicaId: test_utils.rid('test1')
387
- });
388
- await batch.commit('1/1');
389
- });
390
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
391
- const sourceTable = TEST_TABLE;
392
- await batch.markAllSnapshotDone('1/1');
393
- await batch.save({
394
- sourceTable,
395
- tag: storage.SaveOperationTag.DELETE,
396
- beforeReplicaId: test_utils.rid('test1')
397
- });
398
- await batch.commit('2/1');
399
- });
346
+ {
347
+ const env_6 = { stack: [], error: void 0, hasError: false };
348
+ try {
349
+ const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
350
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
351
+ await writer.markAllSnapshotDone('1/1');
352
+ await writer.save({
353
+ sourceTable,
354
+ tag: storage.SaveOperationTag.INSERT,
355
+ after: {
356
+ id: 'test1',
357
+ description: 'test1'
358
+ },
359
+ afterReplicaId: test_utils.rid('test1')
360
+ });
361
+ await writer.save({
362
+ sourceTable,
363
+ tag: storage.SaveOperationTag.DELETE,
364
+ beforeReplicaId: test_utils.rid('test1')
365
+ });
366
+ await writer.save({
367
+ sourceTable,
368
+ tag: storage.SaveOperationTag.INSERT,
369
+ after: {
370
+ id: 'test1',
371
+ description: 'test1'
372
+ },
373
+ afterReplicaId: test_utils.rid('test1')
374
+ });
375
+ await writer.commit('1/1');
376
+ }
377
+ catch (e_5) {
378
+ env_6.error = e_5;
379
+ env_6.hasError = true;
380
+ }
381
+ finally {
382
+ const result_5 = __disposeResources(env_6);
383
+ if (result_5)
384
+ await result_5;
385
+ }
386
+ }
387
+ {
388
+ const env_7 = { stack: [], error: void 0, hasError: false };
389
+ try {
390
+ const writer = __addDisposableResource(env_7, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
391
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
392
+ await writer.markAllSnapshotDone('1/1');
393
+ await writer.save({
394
+ sourceTable,
395
+ tag: storage.SaveOperationTag.DELETE,
396
+ beforeReplicaId: test_utils.rid('test1')
397
+ });
398
+ await writer.commit('2/1');
399
+ }
400
+ catch (e_6) {
401
+ env_7.error = e_6;
402
+ env_7.hasError = true;
403
+ }
404
+ finally {
405
+ const result_6 = __disposeResources(env_7);
406
+ if (result_6)
407
+ await result_6;
408
+ }
409
+ }
400
410
  const { checkpoint } = await bucketStorage.getCheckpoint();
401
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
411
+ const request = bucketRequest(syncRules, 'global[]');
412
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
402
413
  expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
403
414
  const data = batch[0].chunkData.data.map((d) => {
404
415
  return {
@@ -409,20 +420,20 @@ bucket_definitions:
409
420
  });
410
421
  expect(data).toMatchSnapshot();
411
422
  }
412
- catch (e_5) {
413
- env_5.error = e_5;
423
+ catch (e_7) {
424
+ env_5.error = e_7;
414
425
  env_5.hasError = true;
415
426
  }
416
427
  finally {
417
- const result_5 = __disposeResources(env_5);
418
- if (result_5)
419
- await result_5;
428
+ const result_7 = __disposeResources(env_5);
429
+ if (result_7)
430
+ await result_7;
420
431
  }
421
432
  });
422
433
  test('changing client ids', async () => {
423
- const env_6 = { stack: [], error: void 0, hasError: false };
434
+ const env_8 = { stack: [], error: void 0, hasError: false };
424
435
  try {
425
- const factory = __addDisposableResource(env_6, await generateStorageFactory(), true);
436
+ const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
426
437
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
427
438
  bucket_definitions:
428
439
  global:
@@ -432,43 +443,42 @@ bucket_definitions:
432
443
  storageVersion
433
444
  }));
434
445
  const bucketStorage = factory.getInstance(syncRules);
435
- const sourceTable = TEST_TABLE;
436
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
437
- await batch.markAllSnapshotDone('1/1');
438
- await batch.save({
439
- sourceTable,
440
- tag: storage.SaveOperationTag.INSERT,
441
- after: {
442
- id: 'test1',
443
- client_id: 'client1a',
444
- description: 'test1a'
445
- },
446
- afterReplicaId: test_utils.rid('test1')
447
- });
448
- await batch.save({
449
- sourceTable,
450
- tag: storage.SaveOperationTag.UPDATE,
451
- after: {
452
- id: 'test1',
453
- client_id: 'client1b',
454
- description: 'test1b'
455
- },
456
- afterReplicaId: test_utils.rid('test1')
457
- });
458
- await batch.save({
459
- sourceTable,
460
- tag: storage.SaveOperationTag.INSERT,
461
- after: {
462
- id: 'test2',
463
- client_id: 'client2',
464
- description: 'test2'
465
- },
466
- afterReplicaId: test_utils.rid('test2')
467
- });
468
- await batch.commit('1/1');
446
+ const writer = __addDisposableResource(env_8, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
447
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
448
+ await writer.markAllSnapshotDone('1/1');
449
+ await writer.save({
450
+ sourceTable,
451
+ tag: storage.SaveOperationTag.INSERT,
452
+ after: {
453
+ id: 'test1',
454
+ client_id: 'client1a',
455
+ description: 'test1a'
456
+ },
457
+ afterReplicaId: test_utils.rid('test1')
458
+ });
459
+ await writer.save({
460
+ sourceTable,
461
+ tag: storage.SaveOperationTag.UPDATE,
462
+ after: {
463
+ id: 'test1',
464
+ client_id: 'client1b',
465
+ description: 'test1b'
466
+ },
467
+ afterReplicaId: test_utils.rid('test1')
469
468
  });
469
+ await writer.save({
470
+ sourceTable,
471
+ tag: storage.SaveOperationTag.INSERT,
472
+ after: {
473
+ id: 'test2',
474
+ client_id: 'client2',
475
+ description: 'test2'
476
+ },
477
+ afterReplicaId: test_utils.rid('test2')
478
+ });
479
+ await writer.commit('1/1');
470
480
  const { checkpoint } = await bucketStorage.getCheckpoint();
471
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
481
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
472
482
  const data = batch[0].chunkData.data.map((d) => {
473
483
  return {
474
484
  op: d.op,
@@ -482,20 +492,20 @@ bucket_definitions:
482
492
  { op: 'PUT', object_id: 'client2' }
483
493
  ]);
484
494
  }
485
- catch (e_6) {
486
- env_6.error = e_6;
487
- env_6.hasError = true;
495
+ catch (e_8) {
496
+ env_8.error = e_8;
497
+ env_8.hasError = true;
488
498
  }
489
499
  finally {
490
- const result_6 = __disposeResources(env_6);
491
- if (result_6)
492
- await result_6;
500
+ const result_8 = __disposeResources(env_8);
501
+ if (result_8)
502
+ await result_8;
493
503
  }
494
504
  });
495
505
  test('re-apply delete', async () => {
496
- const env_7 = { stack: [], error: void 0, hasError: false };
506
+ const env_9 = { stack: [], error: void 0, hasError: false };
497
507
  try {
498
- const factory = __addDisposableResource(env_7, await generateStorageFactory(), true);
508
+ const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
499
509
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
500
510
  bucket_definitions:
501
511
  global:
@@ -505,38 +515,34 @@ bucket_definitions:
505
515
  storageVersion
506
516
  }));
507
517
  const bucketStorage = factory.getInstance(syncRules);
508
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
509
- const sourceTable = TEST_TABLE;
510
- await batch.markAllSnapshotDone('1/1');
511
- await batch.save({
512
- sourceTable,
513
- tag: storage.SaveOperationTag.INSERT,
514
- after: {
515
- id: 'test1',
516
- description: 'test1'
517
- },
518
- afterReplicaId: test_utils.rid('test1')
519
- });
518
+ const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
519
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
520
+ await writer.markAllSnapshotDone('1/1');
521
+ await writer.save({
522
+ sourceTable,
523
+ tag: storage.SaveOperationTag.INSERT,
524
+ after: {
525
+ id: 'test1',
526
+ description: 'test1'
527
+ },
528
+ afterReplicaId: test_utils.rid('test1')
520
529
  });
521
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
522
- const sourceTable = TEST_TABLE;
523
- await batch.save({
524
- sourceTable,
525
- tag: storage.SaveOperationTag.DELETE,
526
- beforeReplicaId: test_utils.rid('test1')
527
- });
528
- await batch.commit('1/1');
530
+ await writer.flush();
531
+ await writer.save({
532
+ sourceTable,
533
+ tag: storage.SaveOperationTag.DELETE,
534
+ beforeReplicaId: test_utils.rid('test1')
529
535
  });
530
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
531
- const sourceTable = TEST_TABLE;
532
- await batch.save({
533
- sourceTable,
534
- tag: storage.SaveOperationTag.DELETE,
535
- beforeReplicaId: test_utils.rid('test1')
536
- });
536
+ await writer.commit('1/1');
537
+ await writer.save({
538
+ sourceTable,
539
+ tag: storage.SaveOperationTag.DELETE,
540
+ beforeReplicaId: test_utils.rid('test1')
537
541
  });
542
+ await writer.flush();
538
543
  const { checkpoint } = await bucketStorage.getCheckpoint();
539
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
544
+ const request = bucketRequest(syncRules, 'global[]');
545
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
540
546
  const data = batch[0].chunkData.data.map((d) => {
541
547
  return {
542
548
  op: d.op,
@@ -550,9 +556,7 @@ bucket_definitions:
550
556
  { op: 'PUT', object_id: 'test1', checksum: c1 },
551
557
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
552
558
  ]);
553
- const checksums = [
554
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
555
- ];
559
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
556
560
  expect(checksums).toEqual([
557
561
  {
558
562
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -561,20 +565,20 @@ bucket_definitions:
561
565
  }
562
566
  ]);
563
567
  }
564
- catch (e_7) {
565
- env_7.error = e_7;
566
- env_7.hasError = true;
568
+ catch (e_9) {
569
+ env_9.error = e_9;
570
+ env_9.hasError = true;
567
571
  }
568
572
  finally {
569
- const result_7 = __disposeResources(env_7);
570
- if (result_7)
571
- await result_7;
573
+ const result_9 = __disposeResources(env_9);
574
+ if (result_9)
575
+ await result_9;
572
576
  }
573
577
  });
574
578
  test('re-apply update + delete', async () => {
575
- const env_8 = { stack: [], error: void 0, hasError: false };
579
+ const env_10 = { stack: [], error: void 0, hasError: false };
576
580
  try {
577
- const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
581
+ const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
578
582
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
579
583
  bucket_definitions:
580
584
  global:
@@ -582,77 +586,72 @@ bucket_definitions:
582
586
  - SELECT id, description FROM "%"
583
587
  `, { storageVersion }));
584
588
  const bucketStorage = factory.getInstance(syncRules);
585
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
586
- await batch.markAllSnapshotDone('1/1');
587
- const sourceTable = TEST_TABLE;
588
- await batch.save({
589
- sourceTable,
590
- tag: storage.SaveOperationTag.INSERT,
591
- after: {
592
- id: 'test1',
593
- description: 'test1'
594
- },
595
- afterReplicaId: test_utils.rid('test1')
596
- });
589
+ const writer = __addDisposableResource(env_10, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
590
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
591
+ await writer.markAllSnapshotDone('1/1');
592
+ await writer.save({
593
+ sourceTable,
594
+ tag: storage.SaveOperationTag.INSERT,
595
+ after: {
596
+ id: 'test1',
597
+ description: 'test1'
598
+ },
599
+ afterReplicaId: test_utils.rid('test1')
597
600
  });
598
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
599
- await batch.markAllSnapshotDone('1/1');
600
- const sourceTable = TEST_TABLE;
601
- await batch.save({
602
- sourceTable,
603
- tag: storage.SaveOperationTag.UPDATE,
604
- after: {
605
- id: 'test1',
606
- description: undefined
607
- },
608
- afterReplicaId: test_utils.rid('test1')
609
- });
610
- await batch.save({
611
- sourceTable,
612
- tag: storage.SaveOperationTag.UPDATE,
613
- after: {
614
- id: 'test1',
615
- description: undefined
616
- },
617
- afterReplicaId: test_utils.rid('test1')
618
- });
619
- await batch.save({
620
- sourceTable,
621
- tag: storage.SaveOperationTag.DELETE,
622
- beforeReplicaId: test_utils.rid('test1')
623
- });
624
- await batch.commit('1/1');
601
+ await writer.flush();
602
+ await writer.markAllSnapshotDone('1/1');
603
+ await writer.save({
604
+ sourceTable,
605
+ tag: storage.SaveOperationTag.UPDATE,
606
+ after: {
607
+ id: 'test1',
608
+ description: undefined
609
+ },
610
+ afterReplicaId: test_utils.rid('test1')
625
611
  });
626
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
627
- await batch.markAllSnapshotDone('1/1');
628
- const sourceTable = TEST_TABLE;
629
- await batch.save({
630
- sourceTable,
631
- tag: storage.SaveOperationTag.UPDATE,
632
- after: {
633
- id: 'test1',
634
- description: undefined
635
- },
636
- afterReplicaId: test_utils.rid('test1')
637
- });
638
- await batch.save({
639
- sourceTable,
640
- tag: storage.SaveOperationTag.UPDATE,
641
- after: {
642
- id: 'test1',
643
- description: undefined
644
- },
645
- afterReplicaId: test_utils.rid('test1')
646
- });
647
- await batch.save({
648
- sourceTable,
649
- tag: storage.SaveOperationTag.DELETE,
650
- beforeReplicaId: test_utils.rid('test1')
651
- });
652
- await batch.commit('2/1');
612
+ await writer.save({
613
+ sourceTable,
614
+ tag: storage.SaveOperationTag.UPDATE,
615
+ after: {
616
+ id: 'test1',
617
+ description: undefined
618
+ },
619
+ afterReplicaId: test_utils.rid('test1')
620
+ });
621
+ await writer.save({
622
+ sourceTable,
623
+ tag: storage.SaveOperationTag.DELETE,
624
+ beforeReplicaId: test_utils.rid('test1')
625
+ });
626
+ await writer.commit('1/1');
627
+ await writer.markAllSnapshotDone('1/1');
628
+ await writer.save({
629
+ sourceTable,
630
+ tag: storage.SaveOperationTag.UPDATE,
631
+ after: {
632
+ id: 'test1',
633
+ description: undefined
634
+ },
635
+ afterReplicaId: test_utils.rid('test1')
653
636
  });
637
+ await writer.save({
638
+ sourceTable,
639
+ tag: storage.SaveOperationTag.UPDATE,
640
+ after: {
641
+ id: 'test1',
642
+ description: undefined
643
+ },
644
+ afterReplicaId: test_utils.rid('test1')
645
+ });
646
+ await writer.save({
647
+ sourceTable,
648
+ tag: storage.SaveOperationTag.DELETE,
649
+ beforeReplicaId: test_utils.rid('test1')
650
+ });
651
+ await writer.commit('2/1');
654
652
  const { checkpoint } = await bucketStorage.getCheckpoint();
655
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
653
+ const request = bucketRequest(syncRules, 'global[]');
654
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
656
655
  const data = batch[0].chunkData.data.map((d) => {
657
656
  return {
658
657
  op: d.op,
@@ -668,9 +667,7 @@ bucket_definitions:
668
667
  { op: 'PUT', object_id: 'test1', checksum: c1 },
669
668
  { op: 'REMOVE', object_id: 'test1', checksum: c2 }
670
669
  ]);
671
- const checksums = [
672
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
673
- ];
670
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
674
671
  expect(checksums).toEqual([
675
672
  {
676
673
  bucket: bucketRequest(syncRules, 'global[]').bucket,
@@ -679,18 +676,18 @@ bucket_definitions:
679
676
  }
680
677
  ]);
681
678
  }
682
- catch (e_8) {
683
- env_8.error = e_8;
684
- env_8.hasError = true;
679
+ catch (e_10) {
680
+ env_10.error = e_10;
681
+ env_10.hasError = true;
685
682
  }
686
683
  finally {
687
- const result_8 = __disposeResources(env_8);
688
- if (result_8)
689
- await result_8;
684
+ const result_10 = __disposeResources(env_10);
685
+ if (result_10)
686
+ await result_10;
690
687
  }
691
688
  });
692
689
  test('batch with overlapping replica ids', async () => {
693
- const env_9 = { stack: [], error: void 0, hasError: false };
690
+ const env_11 = { stack: [], error: void 0, hasError: false };
694
691
  try {
695
692
  // This test checks that we get the correct output when processing rows with:
696
693
  // 1. changing replica ids
@@ -699,7 +696,7 @@ bucket_definitions:
699
696
  // It can break at two places:
700
697
  // 1. Not getting the correct "current_data" state for each operation.
701
698
  // 2. Output order not being correct.
702
- const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
699
+ const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
703
700
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
704
701
  bucket_definitions:
705
702
  global:
@@ -707,105 +704,104 @@ bucket_definitions:
707
704
  - SELECT id, description FROM "test"
708
705
  `, { storageVersion }));
709
706
  const bucketStorage = factory.getInstance(syncRules);
707
+ const writer = __addDisposableResource(env_11, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
708
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
710
709
  // Pre-setup
711
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
712
- await batch.markAllSnapshotDone('1/1');
713
- const sourceTable = TEST_TABLE;
714
- await batch.save({
715
- sourceTable,
716
- tag: storage.SaveOperationTag.INSERT,
717
- after: {
718
- id: 'test1',
719
- description: 'test1a'
720
- },
721
- afterReplicaId: test_utils.rid('test1')
722
- });
723
- await batch.save({
724
- sourceTable,
725
- tag: storage.SaveOperationTag.INSERT,
726
- after: {
727
- id: 'test2',
728
- description: 'test2a'
729
- },
730
- afterReplicaId: test_utils.rid('test2')
731
- });
710
+ await writer.markAllSnapshotDone('1/1');
711
+ await writer.save({
712
+ sourceTable,
713
+ tag: storage.SaveOperationTag.INSERT,
714
+ after: {
715
+ id: 'test1',
716
+ description: 'test1a'
717
+ },
718
+ afterReplicaId: test_utils.rid('test1')
732
719
  });
720
+ await writer.save({
721
+ sourceTable,
722
+ tag: storage.SaveOperationTag.INSERT,
723
+ after: {
724
+ id: 'test2',
725
+ description: 'test2a'
726
+ },
727
+ afterReplicaId: test_utils.rid('test2')
728
+ });
729
+ const result1 = await writer.flush();
733
730
  const checkpoint1 = result1?.flushed_op ?? 0n;
734
731
  // Test batch
735
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
736
- const sourceTable = TEST_TABLE;
737
- // b
738
- await batch.save({
739
- sourceTable,
740
- tag: storage.SaveOperationTag.INSERT,
741
- after: {
742
- id: 'test1',
743
- description: 'test1b'
744
- },
745
- afterReplicaId: test_utils.rid('test1')
746
- });
747
- await batch.save({
748
- sourceTable,
749
- tag: storage.SaveOperationTag.UPDATE,
750
- before: {
751
- id: 'test1'
752
- },
753
- beforeReplicaId: test_utils.rid('test1'),
754
- after: {
755
- id: 'test2',
756
- description: 'test2b'
757
- },
758
- afterReplicaId: test_utils.rid('test2')
759
- });
760
- await batch.save({
761
- sourceTable,
762
- tag: storage.SaveOperationTag.UPDATE,
763
- before: {
764
- id: 'test2'
765
- },
766
- beforeReplicaId: test_utils.rid('test2'),
767
- after: {
768
- id: 'test3',
769
- description: 'test3b'
770
- },
771
- afterReplicaId: test_utils.rid('test3')
772
- });
773
- // c
774
- await batch.save({
775
- sourceTable,
776
- tag: storage.SaveOperationTag.UPDATE,
777
- after: {
778
- id: 'test2',
779
- description: 'test2c'
780
- },
781
- afterReplicaId: test_utils.rid('test2')
782
- });
783
- // d
784
- await batch.save({
785
- sourceTable,
786
- tag: storage.SaveOperationTag.INSERT,
787
- after: {
788
- id: 'test4',
789
- description: 'test4d'
790
- },
791
- afterReplicaId: test_utils.rid('test4')
792
- });
793
- await batch.save({
794
- sourceTable,
795
- tag: storage.SaveOperationTag.UPDATE,
796
- before: {
797
- id: 'test4'
798
- },
799
- beforeReplicaId: test_utils.rid('test4'),
800
- after: {
801
- id: 'test5',
802
- description: 'test5d'
803
- },
804
- afterReplicaId: test_utils.rid('test5')
805
- });
732
+ // b
733
+ await writer.save({
734
+ sourceTable,
735
+ tag: storage.SaveOperationTag.INSERT,
736
+ after: {
737
+ id: 'test1',
738
+ description: 'test1b'
739
+ },
740
+ afterReplicaId: test_utils.rid('test1')
741
+ });
742
+ await writer.save({
743
+ sourceTable,
744
+ tag: storage.SaveOperationTag.UPDATE,
745
+ before: {
746
+ id: 'test1'
747
+ },
748
+ beforeReplicaId: test_utils.rid('test1'),
749
+ after: {
750
+ id: 'test2',
751
+ description: 'test2b'
752
+ },
753
+ afterReplicaId: test_utils.rid('test2')
754
+ });
755
+ await writer.save({
756
+ sourceTable,
757
+ tag: storage.SaveOperationTag.UPDATE,
758
+ before: {
759
+ id: 'test2'
760
+ },
761
+ beforeReplicaId: test_utils.rid('test2'),
762
+ after: {
763
+ id: 'test3',
764
+ description: 'test3b'
765
+ },
766
+ afterReplicaId: test_utils.rid('test3')
767
+ });
768
+ // c
769
+ await writer.save({
770
+ sourceTable,
771
+ tag: storage.SaveOperationTag.UPDATE,
772
+ after: {
773
+ id: 'test2',
774
+ description: 'test2c'
775
+ },
776
+ afterReplicaId: test_utils.rid('test2')
777
+ });
778
+ // d
779
+ await writer.save({
780
+ sourceTable,
781
+ tag: storage.SaveOperationTag.INSERT,
782
+ after: {
783
+ id: 'test4',
784
+ description: 'test4d'
785
+ },
786
+ afterReplicaId: test_utils.rid('test4')
806
787
  });
788
+ await writer.save({
789
+ sourceTable,
790
+ tag: storage.SaveOperationTag.UPDATE,
791
+ before: {
792
+ id: 'test4'
793
+ },
794
+ beforeReplicaId: test_utils.rid('test4'),
795
+ after: {
796
+ id: 'test5',
797
+ description: 'test5d'
798
+ },
799
+ afterReplicaId: test_utils.rid('test5')
800
+ });
801
+ const result2 = await writer.flush();
807
802
  const checkpoint2 = result2.flushed_op;
808
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', checkpoint1]])));
803
+ const request = bucketRequest(syncRules, 'global[]', checkpoint1);
804
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
809
805
  const data = batch[0].chunkData.data.map((d) => {
810
806
  return {
811
807
  op: d.op,
@@ -829,18 +825,18 @@ bucket_definitions:
829
825
  { op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
830
826
  ]);
831
827
  }
832
- catch (e_9) {
833
- env_9.error = e_9;
834
- env_9.hasError = true;
828
+ catch (e_11) {
829
+ env_11.error = e_11;
830
+ env_11.hasError = true;
835
831
  }
836
832
  finally {
837
- const result_9 = __disposeResources(env_9);
838
- if (result_9)
839
- await result_9;
833
+ const result_11 = __disposeResources(env_11);
834
+ if (result_11)
835
+ await result_11;
840
836
  }
841
837
  });
842
838
  test('changed data with replica identity full', async () => {
843
- const env_10 = { stack: [], error: void 0, hasError: false };
839
+ const env_12 = { stack: [], error: void 0, hasError: false };
844
840
  try {
845
841
  function rid2(id, description) {
846
842
  return getUuidReplicaIdentityBson({ id, description }, [
@@ -848,7 +844,7 @@ bucket_definitions:
848
844
  { name: 'description', type: 'VARCHAR', typeId: 25 }
849
845
  ]);
850
846
  }
851
- const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
847
+ const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
852
848
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
853
849
  bucket_definitions:
854
850
  global:
@@ -858,53 +854,52 @@ bucket_definitions:
858
854
  storageVersion
859
855
  }));
860
856
  const bucketStorage = factory.getInstance(syncRules);
857
+ const writer = __addDisposableResource(env_12, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
861
858
  const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
862
859
  // Pre-setup
863
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
864
- await batch.markAllSnapshotDone('1/1');
865
- await batch.save({
866
- sourceTable,
867
- tag: storage.SaveOperationTag.INSERT,
868
- after: {
869
- id: 'test1',
870
- description: 'test1a'
871
- },
872
- afterReplicaId: rid2('test1', 'test1a')
873
- });
860
+ await writer.markAllSnapshotDone('1/1');
861
+ await writer.save({
862
+ sourceTable,
863
+ tag: storage.SaveOperationTag.INSERT,
864
+ after: {
865
+ id: 'test1',
866
+ description: 'test1a'
867
+ },
868
+ afterReplicaId: rid2('test1', 'test1a')
874
869
  });
870
+ const result1 = await writer.flush();
875
871
  const checkpoint1 = result1?.flushed_op ?? 0n;
876
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
877
- // Unchanged, but has a before id
878
- await batch.save({
879
- sourceTable,
880
- tag: storage.SaveOperationTag.UPDATE,
881
- before: {
882
- id: 'test1',
883
- description: 'test1a'
884
- },
885
- beforeReplicaId: rid2('test1', 'test1a'),
886
- after: {
887
- id: 'test1',
888
- description: 'test1b'
889
- },
890
- afterReplicaId: rid2('test1', 'test1b')
891
- });
872
+ // Unchanged, but has a before id
873
+ await writer.save({
874
+ sourceTable,
875
+ tag: storage.SaveOperationTag.UPDATE,
876
+ before: {
877
+ id: 'test1',
878
+ description: 'test1a'
879
+ },
880
+ beforeReplicaId: rid2('test1', 'test1a'),
881
+ after: {
882
+ id: 'test1',
883
+ description: 'test1b'
884
+ },
885
+ afterReplicaId: rid2('test1', 'test1b')
892
886
  });
893
- const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
894
- // Delete
895
- await batch.save({
896
- sourceTable,
897
- tag: storage.SaveOperationTag.DELETE,
898
- before: {
899
- id: 'test1',
900
- description: 'test1b'
901
- },
902
- beforeReplicaId: rid2('test1', 'test1b'),
903
- after: undefined
904
- });
887
+ const result2 = await writer.flush();
888
+ // Delete
889
+ await writer.save({
890
+ sourceTable,
891
+ tag: storage.SaveOperationTag.DELETE,
892
+ before: {
893
+ id: 'test1',
894
+ description: 'test1b'
895
+ },
896
+ beforeReplicaId: rid2('test1', 'test1b'),
897
+ after: undefined
905
898
  });
899
+ const result3 = await writer.flush();
906
900
  const checkpoint3 = result3.flushed_op;
907
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])));
901
+ const request = bucketRequest(syncRules, 'global[]');
902
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
908
903
  const data = batch[0].chunkData.data.map((d) => {
909
904
  return {
910
905
  op: d.op,
@@ -938,18 +933,18 @@ bucket_definitions:
938
933
  }
939
934
  ]);
940
935
  }
941
- catch (e_10) {
942
- env_10.error = e_10;
943
- env_10.hasError = true;
936
+ catch (e_12) {
937
+ env_12.error = e_12;
938
+ env_12.hasError = true;
944
939
  }
945
940
  finally {
946
- const result_10 = __disposeResources(env_10);
947
- if (result_10)
948
- await result_10;
941
+ const result_12 = __disposeResources(env_12);
942
+ if (result_12)
943
+ await result_12;
949
944
  }
950
945
  });
951
946
  test('unchanged data with replica identity full', async () => {
952
- const env_11 = { stack: [], error: void 0, hasError: false };
947
+ const env_13 = { stack: [], error: void 0, hasError: false };
953
948
  try {
954
949
  function rid2(id, description) {
955
950
  return getUuidReplicaIdentityBson({ id, description }, [
@@ -957,7 +952,7 @@ bucket_definitions:
957
952
  { name: 'description', type: 'VARCHAR', typeId: 25 }
958
953
  ]);
959
954
  }
960
- const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
955
+ const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
961
956
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
962
957
  bucket_definitions:
963
958
  global:
@@ -967,53 +962,52 @@ bucket_definitions:
967
962
  storageVersion
968
963
  }));
969
964
  const bucketStorage = factory.getInstance(syncRules);
965
+ const writer = __addDisposableResource(env_13, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
970
966
  const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
971
967
  // Pre-setup
972
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
973
- await batch.markAllSnapshotDone('1/1');
974
- await batch.save({
975
- sourceTable,
976
- tag: storage.SaveOperationTag.INSERT,
977
- after: {
978
- id: 'test1',
979
- description: 'test1a'
980
- },
981
- afterReplicaId: rid2('test1', 'test1a')
982
- });
968
+ await writer.markAllSnapshotDone('1/1');
969
+ await writer.save({
970
+ sourceTable,
971
+ tag: storage.SaveOperationTag.INSERT,
972
+ after: {
973
+ id: 'test1',
974
+ description: 'test1a'
975
+ },
976
+ afterReplicaId: rid2('test1', 'test1a')
983
977
  });
978
+ const result1 = await writer.flush();
984
979
  const checkpoint1 = result1?.flushed_op ?? 0n;
985
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
986
- // Unchanged, but has a before id
987
- await batch.save({
988
- sourceTable,
989
- tag: storage.SaveOperationTag.UPDATE,
990
- before: {
991
- id: 'test1',
992
- description: 'test1a'
993
- },
994
- beforeReplicaId: rid2('test1', 'test1a'),
995
- after: {
996
- id: 'test1',
997
- description: 'test1a'
998
- },
999
- afterReplicaId: rid2('test1', 'test1a')
1000
- });
1001
- });
1002
- const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1003
- // Delete
1004
- await batch.save({
1005
- sourceTable,
1006
- tag: storage.SaveOperationTag.DELETE,
1007
- before: {
1008
- id: 'test1',
1009
- description: 'test1a'
1010
- },
1011
- beforeReplicaId: rid2('test1', 'test1a'),
1012
- after: undefined
1013
- });
980
+ // Unchanged, but has a before id
981
+ await writer.save({
982
+ sourceTable,
983
+ tag: storage.SaveOperationTag.UPDATE,
984
+ before: {
985
+ id: 'test1',
986
+ description: 'test1a'
987
+ },
988
+ beforeReplicaId: rid2('test1', 'test1a'),
989
+ after: {
990
+ id: 'test1',
991
+ description: 'test1a'
992
+ },
993
+ afterReplicaId: rid2('test1', 'test1a')
994
+ });
995
+ const result2 = await writer.flush();
996
+ // Delete
997
+ await writer.save({
998
+ sourceTable,
999
+ tag: storage.SaveOperationTag.DELETE,
1000
+ before: {
1001
+ id: 'test1',
1002
+ description: 'test1a'
1003
+ },
1004
+ beforeReplicaId: rid2('test1', 'test1a'),
1005
+ after: undefined
1014
1006
  });
1007
+ const result3 = await writer.flush();
1015
1008
  const checkpoint3 = result3.flushed_op;
1016
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])));
1009
+ const request = bucketRequest(syncRules, 'global[]');
1010
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
1017
1011
  const data = batch[0].chunkData.data.map((d) => {
1018
1012
  return {
1019
1013
  op: d.op,
@@ -1040,24 +1034,24 @@ bucket_definitions:
1040
1034
  }
1041
1035
  ]);
1042
1036
  }
1043
- catch (e_11) {
1044
- env_11.error = e_11;
1045
- env_11.hasError = true;
1037
+ catch (e_13) {
1038
+ env_13.error = e_13;
1039
+ env_13.hasError = true;
1046
1040
  }
1047
1041
  finally {
1048
- const result_11 = __disposeResources(env_11);
1049
- if (result_11)
1050
- await result_11;
1042
+ const result_13 = __disposeResources(env_13);
1043
+ if (result_13)
1044
+ await result_13;
1051
1045
  }
1052
1046
  });
1053
1047
  test('large batch', async () => {
1054
- const env_12 = { stack: [], error: void 0, hasError: false };
1048
+ const env_14 = { stack: [], error: void 0, hasError: false };
1055
1049
  try {
1056
1050
  // Test syncing a batch of data that is small in count,
1057
1051
  // but large enough in size to be split over multiple returned batches.
1058
1052
  // The specific batch splits is an implementation detail of the storage driver,
1059
1053
  // and the test will have to updated when other implementations are added.
1060
- const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
1054
+ const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
1061
1055
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1062
1056
  bucket_definitions:
1063
1057
  global:
@@ -1067,54 +1061,54 @@ bucket_definitions:
1067
1061
  storageVersion
1068
1062
  }));
1069
1063
  const bucketStorage = factory.getInstance(syncRules);
1070
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1071
- await batch.markAllSnapshotDone('1/1');
1072
- const sourceTable = TEST_TABLE;
1073
- const largeDescription = '0123456789'.repeat(12_000_00);
1074
- await batch.save({
1075
- sourceTable,
1076
- tag: storage.SaveOperationTag.INSERT,
1077
- after: {
1078
- id: 'test1',
1079
- description: 'test1'
1080
- },
1081
- afterReplicaId: test_utils.rid('test1')
1082
- });
1083
- await batch.save({
1084
- sourceTable,
1085
- tag: storage.SaveOperationTag.INSERT,
1086
- after: {
1087
- id: 'large1',
1088
- description: largeDescription
1089
- },
1090
- afterReplicaId: test_utils.rid('large1')
1091
- });
1092
- // Large enough to split the returned batch
1093
- await batch.save({
1094
- sourceTable,
1095
- tag: storage.SaveOperationTag.INSERT,
1096
- after: {
1097
- id: 'large2',
1098
- description: largeDescription
1099
- },
1100
- afterReplicaId: test_utils.rid('large2')
1101
- });
1102
- await batch.save({
1103
- sourceTable,
1104
- tag: storage.SaveOperationTag.INSERT,
1105
- after: {
1106
- id: 'test3',
1107
- description: 'test3'
1108
- },
1109
- afterReplicaId: test_utils.rid('test3')
1110
- });
1111
- await batch.commit('1/1');
1064
+ const writer = __addDisposableResource(env_14, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1065
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1066
+ await writer.markAllSnapshotDone('1/1');
1067
+ const largeDescription = '0123456789'.repeat(12_000_00);
1068
+ await writer.save({
1069
+ sourceTable,
1070
+ tag: storage.SaveOperationTag.INSERT,
1071
+ after: {
1072
+ id: 'test1',
1073
+ description: 'test1'
1074
+ },
1075
+ afterReplicaId: test_utils.rid('test1')
1076
+ });
1077
+ await writer.save({
1078
+ sourceTable,
1079
+ tag: storage.SaveOperationTag.INSERT,
1080
+ after: {
1081
+ id: 'large1',
1082
+ description: largeDescription
1083
+ },
1084
+ afterReplicaId: test_utils.rid('large1')
1112
1085
  });
1086
+ // Large enough to split the returned batch
1087
+ await writer.save({
1088
+ sourceTable,
1089
+ tag: storage.SaveOperationTag.INSERT,
1090
+ after: {
1091
+ id: 'large2',
1092
+ description: largeDescription
1093
+ },
1094
+ afterReplicaId: test_utils.rid('large2')
1095
+ });
1096
+ await writer.save({
1097
+ sourceTable,
1098
+ tag: storage.SaveOperationTag.INSERT,
1099
+ after: {
1100
+ id: 'test3',
1101
+ description: 'test3'
1102
+ },
1103
+ afterReplicaId: test_utils.rid('test3')
1104
+ });
1105
+ await writer.commit('1/1');
1113
1106
  const { checkpoint } = await bucketStorage.getCheckpoint();
1114
1107
  const options = {
1115
1108
  chunkLimitBytes: 16 * 1024 * 1024
1116
1109
  };
1117
- const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options));
1110
+ const request = bucketRequest(syncRules, 'global[]');
1111
+ const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
1118
1112
  expect(test_utils.getBatchData(batch1)).toEqual([
1119
1113
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1120
1114
  { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
@@ -1124,7 +1118,7 @@ bucket_definitions:
1124
1118
  has_more: true,
1125
1119
  next_after: '2'
1126
1120
  });
1127
- const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]), options));
1121
+ const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
1128
1122
  expect(test_utils.getBatchData(batch2)).toEqual([
1129
1123
  { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
1130
1124
  { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
@@ -1134,25 +1128,25 @@ bucket_definitions:
1134
1128
  has_more: false,
1135
1129
  next_after: '4'
1136
1130
  });
1137
- const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]), options));
1131
+ const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
1138
1132
  expect(test_utils.getBatchData(batch3)).toEqual([]);
1139
1133
  expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1140
1134
  }
1141
- catch (e_12) {
1142
- env_12.error = e_12;
1143
- env_12.hasError = true;
1135
+ catch (e_14) {
1136
+ env_14.error = e_14;
1137
+ env_14.hasError = true;
1144
1138
  }
1145
1139
  finally {
1146
- const result_12 = __disposeResources(env_12);
1147
- if (result_12)
1148
- await result_12;
1140
+ const result_14 = __disposeResources(env_14);
1141
+ if (result_14)
1142
+ await result_14;
1149
1143
  }
1150
1144
  });
1151
1145
  test('long batch', async () => {
1152
- const env_13 = { stack: [], error: void 0, hasError: false };
1146
+ const env_15 = { stack: [], error: void 0, hasError: false };
1153
1147
  try {
1154
1148
  // Test syncing a batch of data that is limited by count.
1155
- const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
1149
+ const factory = __addDisposableResource(env_15, await generateStorageFactory(), true);
1156
1150
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1157
1151
  bucket_definitions:
1158
1152
  global:
@@ -1162,24 +1156,24 @@ bucket_definitions:
1162
1156
  storageVersion
1163
1157
  }));
1164
1158
  const bucketStorage = factory.getInstance(syncRules);
1165
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1166
- await batch.markAllSnapshotDone('1/1');
1167
- const sourceTable = TEST_TABLE;
1168
- for (let i = 1; i <= 6; i++) {
1169
- await batch.save({
1170
- sourceTable,
1171
- tag: storage.SaveOperationTag.INSERT,
1172
- after: {
1173
- id: `test${i}`,
1174
- description: `test${i}`
1175
- },
1176
- afterReplicaId: `test${i}`
1177
- });
1178
- }
1179
- await batch.commit('1/1');
1180
- });
1159
+ const writer = __addDisposableResource(env_15, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1160
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1161
+ await writer.markAllSnapshotDone('1/1');
1162
+ for (let i = 1; i <= 6; i++) {
1163
+ await writer.save({
1164
+ sourceTable,
1165
+ tag: storage.SaveOperationTag.INSERT,
1166
+ after: {
1167
+ id: `test${i}`,
1168
+ description: `test${i}`
1169
+ },
1170
+ afterReplicaId: `test${i}`
1171
+ });
1172
+ }
1173
+ await writer.commit('1/1');
1181
1174
  const { checkpoint } = await bucketStorage.getCheckpoint();
1182
- const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), { limit: 4 }));
1175
+ const request = bucketRequest(syncRules, 'global[]');
1176
+ const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
1183
1177
  expect(test_utils.getBatchData(batch1)).toEqual([
1184
1178
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1185
1179
  { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
@@ -1191,7 +1185,7 @@ bucket_definitions:
1191
1185
  has_more: true,
1192
1186
  next_after: '4'
1193
1187
  });
1194
- const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch1.chunkData.next_after)]]), {
1188
+ const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
1195
1189
  limit: 4
1196
1190
  }));
1197
1191
  expect(test_utils.getBatchData(batch2)).toEqual([
@@ -1203,27 +1197,27 @@ bucket_definitions:
1203
1197
  has_more: false,
1204
1198
  next_after: '6'
1205
1199
  });
1206
- const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', BigInt(batch2.chunkData.next_after)]]), {
1200
+ const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
1207
1201
  limit: 4
1208
1202
  }));
1209
1203
  expect(test_utils.getBatchData(batch3)).toEqual([]);
1210
1204
  expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1211
1205
  }
1212
- catch (e_13) {
1213
- env_13.error = e_13;
1214
- env_13.hasError = true;
1206
+ catch (e_15) {
1207
+ env_15.error = e_15;
1208
+ env_15.hasError = true;
1215
1209
  }
1216
1210
  finally {
1217
- const result_13 = __disposeResources(env_13);
1218
- if (result_13)
1219
- await result_13;
1211
+ const result_15 = __disposeResources(env_15);
1212
+ if (result_15)
1213
+ await result_15;
1220
1214
  }
1221
1215
  });
1222
1216
  describe('batch has_more', () => {
1223
1217
  const setup = async (options) => {
1224
- const env_14 = { stack: [], error: void 0, hasError: false };
1218
+ const env_16 = { stack: [], error: void 0, hasError: false };
1225
1219
  try {
1226
- const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
1220
+ const factory = __addDisposableResource(env_16, await generateStorageFactory(), true);
1227
1221
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1228
1222
  bucket_definitions:
1229
1223
  global1:
@@ -1234,45 +1228,43 @@ bucket_definitions:
1234
1228
  - SELECT id, description FROM test WHERE bucket = 'global2'
1235
1229
  `, { storageVersion }));
1236
1230
  const bucketStorage = factory.getInstance(syncRules);
1237
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1238
- await batch.markAllSnapshotDone('1/1');
1239
- const sourceTable = TEST_TABLE;
1240
- for (let i = 1; i <= 10; i++) {
1241
- await batch.save({
1242
- sourceTable,
1243
- tag: storage.SaveOperationTag.INSERT,
1244
- after: {
1245
- id: `test${i}`,
1246
- description: `test${i}`,
1247
- bucket: i == 1 ? 'global1' : 'global2'
1248
- },
1249
- afterReplicaId: `test${i}`
1250
- });
1251
- }
1252
- await batch.commit('1/1');
1253
- });
1231
+ const writer = __addDisposableResource(env_16, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1232
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1233
+ await writer.markAllSnapshotDone('1/1');
1234
+ for (let i = 1; i <= 10; i++) {
1235
+ await writer.save({
1236
+ sourceTable,
1237
+ tag: storage.SaveOperationTag.INSERT,
1238
+ after: {
1239
+ id: `test${i}`,
1240
+ description: `test${i}`,
1241
+ bucket: i == 1 ? 'global1' : 'global2'
1242
+ },
1243
+ afterReplicaId: `test${i}`
1244
+ });
1245
+ }
1246
+ await writer.commit('1/1');
1254
1247
  const { checkpoint } = await bucketStorage.getCheckpoint();
1255
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [
1256
- ['global1[]', 0n],
1257
- ['global2[]', 0n]
1258
- ]), options));
1259
- return { syncRules, batch };
1248
+ const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
1249
+ const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
1250
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
1251
+ return { batch, global1Request, global2Request };
1260
1252
  }
1261
- catch (e_14) {
1262
- env_14.error = e_14;
1263
- env_14.hasError = true;
1253
+ catch (e_16) {
1254
+ env_16.error = e_16;
1255
+ env_16.hasError = true;
1264
1256
  }
1265
1257
  finally {
1266
- const result_14 = __disposeResources(env_14);
1267
- if (result_14)
1268
- await result_14;
1258
+ const result_16 = __disposeResources(env_16);
1259
+ if (result_16)
1260
+ await result_16;
1269
1261
  }
1270
1262
  };
1271
1263
  test('batch has_more (1)', async () => {
1272
- const { batch, syncRules } = await setup({ limit: 5 });
1264
+ const { batch, global1Request, global2Request } = await setup({ limit: 5 });
1273
1265
  expect(batch.length).toEqual(2);
1274
- expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
1275
- expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1266
+ expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1267
+ expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1276
1268
  expect(test_utils.getBatchData(batch[0])).toEqual([
1277
1269
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
1278
1270
  ]);
@@ -1294,10 +1286,10 @@ bucket_definitions:
1294
1286
  });
1295
1287
  });
1296
1288
  test('batch has_more (2)', async () => {
1297
- const { batch, syncRules } = await setup({ limit: 11 });
1289
+ const { batch, global1Request, global2Request } = await setup({ limit: 11 });
1298
1290
  expect(batch.length).toEqual(2);
1299
- expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
1300
- expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1291
+ expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1292
+ expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1301
1293
  expect(test_utils.getBatchData(batch[0])).toEqual([
1302
1294
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
1303
1295
  ]);
@@ -1325,11 +1317,11 @@ bucket_definitions:
1325
1317
  });
1326
1318
  test('batch has_more (3)', async () => {
1327
1319
  // 50 bytes is more than 1 row, less than 2 rows
1328
- const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 });
1320
+ const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
1329
1321
  expect(batch.length).toEqual(3);
1330
- expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket);
1331
- expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1332
- expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket);
1322
+ expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
1323
+ expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
1324
+ expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
1333
1325
  expect(test_utils.getBatchData(batch[0])).toEqual([
1334
1326
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
1335
1327
  ]);
@@ -1357,9 +1349,9 @@ bucket_definitions:
1357
1349
  });
1358
1350
  });
1359
1351
  test('empty storage metrics', async () => {
1360
- const env_15 = { stack: [], error: void 0, hasError: false };
1352
+ const env_17 = { stack: [], error: void 0, hasError: false };
1361
1353
  try {
1362
- const f = __addDisposableResource(env_15, await generateStorageFactory({ dropAll: true }), true);
1354
+ const f = __addDisposableResource(env_17, await generateStorageFactory({ dropAll: true }), true);
1363
1355
  const metrics = await f.getStorageMetrics();
1364
1356
  expect(metrics).toEqual({
1365
1357
  operations_size_bytes: 0,
@@ -1368,29 +1360,28 @@ bucket_definitions:
1368
1360
  });
1369
1361
  const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
1370
1362
  const storage = f.getInstance(r.persisted_sync_rules);
1371
- await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1372
- await batch.markAllSnapshotDone('1/0');
1373
- await batch.keepalive('1/0');
1374
- });
1363
+ const writer = __addDisposableResource(env_17, await storage.createWriter(test_utils.BATCH_OPTIONS), true);
1364
+ await writer.markAllSnapshotDone('1/0');
1365
+ await writer.keepalive('1/0');
1375
1366
  await f.getStorageMetrics();
1376
1367
  }
1377
- catch (e_15) {
1378
- env_15.error = e_15;
1379
- env_15.hasError = true;
1368
+ catch (e_17) {
1369
+ env_17.error = e_17;
1370
+ env_17.hasError = true;
1380
1371
  }
1381
1372
  finally {
1382
- const result_15 = __disposeResources(env_15);
1383
- if (result_15)
1384
- await result_15;
1373
+ const result_17 = __disposeResources(env_17);
1374
+ if (result_17)
1375
+ await result_17;
1385
1376
  }
1386
1377
  });
1387
1378
  test('op_id initialization edge case', async () => {
1388
- const env_16 = { stack: [], error: void 0, hasError: false };
1379
+ const env_18 = { stack: [], error: void 0, hasError: false };
1389
1380
  try {
1390
1381
  // Test syncing a batch of data that is small in count,
1391
1382
  // but large enough in size to be split over multiple returned chunks.
1392
1383
  // Similar to the above test, but splits over 1MB chunks.
1393
- const factory = __addDisposableResource(env_16, await generateStorageFactory(), true);
1384
+ const factory = __addDisposableResource(env_18, await generateStorageFactory(), true);
1394
1385
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1395
1386
  bucket_definitions:
1396
1387
  global:
@@ -1401,50 +1392,49 @@ bucket_definitions:
1401
1392
  storageVersion
1402
1393
  }));
1403
1394
  const bucketStorage = factory.getInstance(syncRules);
1404
- const sourceTable = test_utils.makeTestTable('test', ['id'], config);
1405
- const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id'], config);
1406
- const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1407
- await batch.markAllSnapshotDone('1/1');
1408
- // This saves a record to current_data, but not bucket_data.
1409
- // This causes a checkpoint to be created without increasing the op_id sequence.
1410
- await batch.save({
1411
- sourceTable: sourceTableIgnore,
1412
- tag: storage.SaveOperationTag.INSERT,
1413
- after: {
1414
- id: 'test1'
1415
- },
1416
- afterReplicaId: test_utils.rid('test1')
1417
- });
1395
+ const writer = __addDisposableResource(env_18, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1396
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
1397
+ const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
1398
+ await writer.markAllSnapshotDone('1/1');
1399
+ // This saves a record to current_data, but not bucket_data.
1400
+ // This causes a checkpoint to be created without increasing the op_id sequence.
1401
+ await writer.save({
1402
+ sourceTable: sourceTableIgnore,
1403
+ tag: storage.SaveOperationTag.INSERT,
1404
+ after: {
1405
+ id: 'test1'
1406
+ },
1407
+ afterReplicaId: test_utils.rid('test1')
1418
1408
  });
1409
+ const result1 = await writer.flush();
1419
1410
  const checkpoint1 = result1.flushed_op;
1420
- const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1421
- await batch.save({
1422
- sourceTable: sourceTable,
1423
- tag: storage.SaveOperationTag.INSERT,
1424
- after: {
1425
- id: 'test2'
1426
- },
1427
- afterReplicaId: test_utils.rid('test2')
1428
- });
1411
+ await writer.save({
1412
+ sourceTable: sourceTable,
1413
+ tag: storage.SaveOperationTag.INSERT,
1414
+ after: {
1415
+ id: 'test2'
1416
+ },
1417
+ afterReplicaId: test_utils.rid('test2')
1429
1418
  });
1419
+ const result2 = await writer.flush();
1430
1420
  const checkpoint2 = result2.flushed_op;
1431
1421
  // we expect 0n and 1n, or 1n and 2n.
1432
1422
  expect(checkpoint2).toBeGreaterThan(checkpoint1);
1433
1423
  }
1434
- catch (e_16) {
1435
- env_16.error = e_16;
1436
- env_16.hasError = true;
1424
+ catch (e_18) {
1425
+ env_18.error = e_18;
1426
+ env_18.hasError = true;
1437
1427
  }
1438
1428
  finally {
1439
- const result_16 = __disposeResources(env_16);
1440
- if (result_16)
1441
- await result_16;
1429
+ const result_18 = __disposeResources(env_18);
1430
+ if (result_18)
1431
+ await result_18;
1442
1432
  }
1443
1433
  });
1444
1434
  test('unchanged checksums', async () => {
1445
- const env_17 = { stack: [], error: void 0, hasError: false };
1435
+ const env_19 = { stack: [], error: void 0, hasError: false };
1446
1436
  try {
1447
- const factory = __addDisposableResource(env_17, await generateStorageFactory(), true);
1437
+ const factory = __addDisposableResource(env_19, await generateStorageFactory(), true);
1448
1438
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1449
1439
  bucket_definitions:
1450
1440
  global:
@@ -1454,49 +1444,41 @@ bucket_definitions:
1454
1444
  storageVersion
1455
1445
  }));
1456
1446
  const bucketStorage = factory.getInstance(syncRules);
1457
- const sourceTable = TEST_TABLE;
1458
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1459
- await batch.markAllSnapshotDone('1/1');
1460
- await batch.save({
1461
- sourceTable,
1462
- tag: storage.SaveOperationTag.INSERT,
1463
- after: {
1464
- id: 'test1',
1465
- description: 'test1a'
1466
- },
1467
- afterReplicaId: test_utils.rid('test1')
1468
- });
1469
- await batch.commit('1/1');
1447
+ const writer = __addDisposableResource(env_19, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1448
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1449
+ await writer.markAllSnapshotDone('1/1');
1450
+ await writer.save({
1451
+ sourceTable,
1452
+ tag: storage.SaveOperationTag.INSERT,
1453
+ after: {
1454
+ id: 'test1',
1455
+ description: 'test1a'
1456
+ },
1457
+ afterReplicaId: test_utils.rid('test1')
1470
1458
  });
1459
+ await writer.commit('1/1');
1471
1460
  const { checkpoint } = await bucketStorage.getCheckpoint();
1472
- const checksums = [
1473
- ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
1474
- ];
1475
- expect(checksums).toEqual([
1476
- { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
1477
- ]);
1478
- const checksums2 = [
1479
- ...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
1480
- ];
1481
- expect(checksums2).toEqual([
1482
- { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
1483
- ]);
1461
+ const request = bucketRequest(syncRules, 'global[]');
1462
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
1463
+ expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1464
+ const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
1465
+ expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1484
1466
  }
1485
- catch (e_17) {
1486
- env_17.error = e_17;
1487
- env_17.hasError = true;
1467
+ catch (e_19) {
1468
+ env_19.error = e_19;
1469
+ env_19.hasError = true;
1488
1470
  }
1489
1471
  finally {
1490
- const result_17 = __disposeResources(env_17);
1491
- if (result_17)
1492
- await result_17;
1472
+ const result_19 = __disposeResources(env_19);
1473
+ if (result_19)
1474
+ await result_19;
1493
1475
  }
1494
1476
  });
1495
1477
  testChecksumBatching(config);
1496
1478
  test('empty checkpoints (1)', async () => {
1497
- const env_18 = { stack: [], error: void 0, hasError: false };
1479
+ const env_20 = { stack: [], error: void 0, hasError: false };
1498
1480
  try {
1499
- const factory = __addDisposableResource(env_18, await generateStorageFactory(), true);
1481
+ const factory = __addDisposableResource(env_20, await generateStorageFactory(), true);
1500
1482
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1501
1483
  bucket_definitions:
1502
1484
  global:
@@ -1504,38 +1486,37 @@ bucket_definitions:
1504
1486
  - SELECT id, description FROM "%"
1505
1487
  `, { storageVersion }));
1506
1488
  const bucketStorage = factory.getInstance(syncRules);
1507
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1508
- await batch.markAllSnapshotDone('1/1');
1509
- await batch.commit('1/1');
1510
- const cp1 = await bucketStorage.getCheckpoint();
1511
- expect(cp1.lsn).toEqual('1/1');
1512
- await batch.commit('2/1', { createEmptyCheckpoints: true });
1513
- const cp2 = await bucketStorage.getCheckpoint();
1514
- expect(cp2.lsn).toEqual('2/1');
1515
- await batch.keepalive('3/1');
1516
- const cp3 = await bucketStorage.getCheckpoint();
1517
- expect(cp3.lsn).toEqual('3/1');
1518
- // For the last one, we skip creating empty checkpoints
1519
- // This means the LSN stays at 3/1.
1520
- await batch.commit('4/1', { createEmptyCheckpoints: false });
1521
- const cp4 = await bucketStorage.getCheckpoint();
1522
- expect(cp4.lsn).toEqual('3/1');
1523
- });
1489
+ const writer = __addDisposableResource(env_20, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1490
+ await writer.markAllSnapshotDone('1/1');
1491
+ await writer.commit('1/1');
1492
+ const cp1 = await bucketStorage.getCheckpoint();
1493
+ expect(cp1.lsn).toEqual('1/1');
1494
+ await writer.commit('2/1', { createEmptyCheckpoints: true });
1495
+ const cp2 = await bucketStorage.getCheckpoint();
1496
+ expect(cp2.lsn).toEqual('2/1');
1497
+ await writer.keepalive('3/1');
1498
+ const cp3 = await bucketStorage.getCheckpoint();
1499
+ expect(cp3.lsn).toEqual('3/1');
1500
+ // For the last one, we skip creating empty checkpoints
1501
+ // This means the LSN stays at 3/1.
1502
+ await writer.commit('4/1', { createEmptyCheckpoints: false });
1503
+ const cp4 = await bucketStorage.getCheckpoint();
1504
+ expect(cp4.lsn).toEqual('3/1');
1524
1505
  }
1525
- catch (e_18) {
1526
- env_18.error = e_18;
1527
- env_18.hasError = true;
1506
+ catch (e_20) {
1507
+ env_20.error = e_20;
1508
+ env_20.hasError = true;
1528
1509
  }
1529
1510
  finally {
1530
- const result_18 = __disposeResources(env_18);
1531
- if (result_18)
1532
- await result_18;
1511
+ const result_20 = __disposeResources(env_20);
1512
+ if (result_20)
1513
+ await result_20;
1533
1514
  }
1534
1515
  });
1535
1516
  test('empty checkpoints (2)', async () => {
1536
- const env_19 = { stack: [], error: void 0, hasError: false };
1517
+ const env_21 = { stack: [], error: void 0, hasError: false };
1537
1518
  try {
1538
- const factory = __addDisposableResource(env_19, await generateStorageFactory(), true);
1519
+ const factory = __addDisposableResource(env_21, await generateStorageFactory(), true);
1539
1520
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1540
1521
  bucket_definitions:
1541
1522
  global:
@@ -1545,50 +1526,48 @@ bucket_definitions:
1545
1526
  storageVersion
1546
1527
  }));
1547
1528
  const bucketStorage = factory.getInstance(syncRules);
1548
- const sourceTable = TEST_TABLE;
1549
- // We simulate two concurrent batches, but nesting is the easiest way to do this.
1550
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch1) => {
1551
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch2) => {
1552
- await batch1.markAllSnapshotDone('1/1');
1553
- await batch1.commit('1/1');
1554
- await batch1.commit('2/1', { createEmptyCheckpoints: false });
1555
- const cp2 = await bucketStorage.getCheckpoint();
1556
- expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
1557
- await batch2.save({
1558
- sourceTable,
1559
- tag: storage.SaveOperationTag.INSERT,
1560
- after: {
1561
- id: 'test1',
1562
- description: 'test1a'
1563
- },
1564
- afterReplicaId: test_utils.rid('test1')
1565
- });
1566
- // This simulates what happens on a snapshot processor.
1567
- // This may later change to a flush() rather than commit().
1568
- await batch2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
1569
- const cp3 = await bucketStorage.getCheckpoint();
1570
- expect(cp3.lsn).toEqual('1/1'); // Still unchanged
1571
- // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
1572
- await batch1.commit('4/1', { createEmptyCheckpoints: false });
1573
- const cp4 = await bucketStorage.getCheckpoint();
1574
- expect(cp4.lsn).toEqual('4/1');
1575
- });
1529
+ const writer1 = __addDisposableResource(env_21, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1530
+ const writer2 = __addDisposableResource(env_21, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1531
+ const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
1532
+ // We simulate two concurrent batches, but sequential calls are enough for this test.
1533
+ await writer1.markAllSnapshotDone('1/1');
1534
+ await writer1.commit('1/1');
1535
+ await writer1.commit('2/1', { createEmptyCheckpoints: false });
1536
+ const cp2 = await bucketStorage.getCheckpoint();
1537
+ expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
1538
+ await writer2.save({
1539
+ sourceTable,
1540
+ tag: storage.SaveOperationTag.INSERT,
1541
+ after: {
1542
+ id: 'test1',
1543
+ description: 'test1a'
1544
+ },
1545
+ afterReplicaId: test_utils.rid('test1')
1576
1546
  });
1547
+ // This simulates what happens on a snapshot processor.
1548
+ // This may later change to a flush() rather than commit().
1549
+ await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
1550
+ const cp3 = await bucketStorage.getCheckpoint();
1551
+ expect(cp3.lsn).toEqual('1/1'); // Still unchanged
1552
+ // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
1553
+ await writer1.commit('4/1', { createEmptyCheckpoints: false });
1554
+ const cp4 = await bucketStorage.getCheckpoint();
1555
+ expect(cp4.lsn).toEqual('4/1');
1577
1556
  }
1578
- catch (e_19) {
1579
- env_19.error = e_19;
1580
- env_19.hasError = true;
1557
+ catch (e_21) {
1558
+ env_21.error = e_21;
1559
+ env_21.hasError = true;
1581
1560
  }
1582
1561
  finally {
1583
- const result_19 = __disposeResources(env_19);
1584
- if (result_19)
1585
- await result_19;
1562
+ const result_21 = __disposeResources(env_21);
1563
+ if (result_21)
1564
+ await result_21;
1586
1565
  }
1587
1566
  });
1588
1567
  test('empty checkpoints (sync rule activation)', async () => {
1589
- const env_20 = { stack: [], error: void 0, hasError: false };
1568
+ const env_22 = { stack: [], error: void 0, hasError: false };
1590
1569
  try {
1591
- const factory = __addDisposableResource(env_20, await generateStorageFactory(), true);
1570
+ const factory = __addDisposableResource(env_22, await generateStorageFactory(), true);
1592
1571
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1593
1572
  bucket_definitions:
1594
1573
  global:
@@ -1598,54 +1577,47 @@ bucket_definitions:
1598
1577
  storageVersion
1599
1578
  }));
1600
1579
  const bucketStorage = factory.getInstance(syncRules);
1601
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1602
- const result = await batch.commit('1/1', { createEmptyCheckpoints: false });
1603
- expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1604
- // Snapshot is only valid once we reach 3/1
1605
- await batch.markAllSnapshotDone('3/1');
1606
- });
1607
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1608
- // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
1609
- const result = await batch.commit('2/1', { createEmptyCheckpoints: false });
1610
- expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1611
- });
1580
+ const writer = __addDisposableResource(env_22, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1581
+ const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
1582
+ expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1583
+ // Snapshot is only valid once we reach 3/1
1584
+ await writer.markAllSnapshotDone('3/1');
1585
+ // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
1586
+ const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
1587
+ expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1612
1588
  // No empty checkpoint should be created by the commit above.
1613
1589
  const cp1 = await bucketStorage.getCheckpoint();
1614
1590
  expect(cp1.lsn).toEqual(null);
1615
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1616
- // After this commit, the snapshot should be valid.
1617
- // We specifically check that this is done even if createEmptyCheckpoints: false.
1618
- const result = await batch.commit('3/1', { createEmptyCheckpoints: false });
1619
- expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true });
1620
- });
1591
+ // After this commit, the snapshot should be valid.
1592
+ // We specifically check that this is done even if createEmptyCheckpoints: false.
1593
+ const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
1594
+ expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
1621
1595
  // Now, the checkpoint should advance the sync rules active.
1622
1596
  const cp2 = await bucketStorage.getCheckpoint();
1623
1597
  expect(cp2.lsn).toEqual('3/1');
1624
1598
  const activeSyncRules = await factory.getActiveSyncRulesContent();
1625
1599
  expect(activeSyncRules?.id).toEqual(syncRules.id);
1626
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1627
- // At this point, it should be a truely empty checkpoint
1628
- const result = await batch.commit('4/1', { createEmptyCheckpoints: false });
1629
- expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false });
1630
- });
1600
+ // At this point, it should be a truely empty checkpoint
1601
+ const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
1602
+ expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
1631
1603
  // Unchanged
1632
1604
  const cp3 = await bucketStorage.getCheckpoint();
1633
1605
  expect(cp3.lsn).toEqual('3/1');
1634
1606
  }
1635
- catch (e_20) {
1636
- env_20.error = e_20;
1637
- env_20.hasError = true;
1607
+ catch (e_22) {
1608
+ env_22.error = e_22;
1609
+ env_22.hasError = true;
1638
1610
  }
1639
1611
  finally {
1640
- const result_20 = __disposeResources(env_20);
1641
- if (result_20)
1642
- await result_20;
1612
+ const result_22 = __disposeResources(env_22);
1613
+ if (result_22)
1614
+ await result_22;
1643
1615
  }
1644
1616
  });
1645
1617
  test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
1646
- const env_21 = { stack: [], error: void 0, hasError: false };
1618
+ const env_23 = { stack: [], error: void 0, hasError: false };
1647
1619
  try {
1648
- const factory = __addDisposableResource(env_21, await generateStorageFactory(), true);
1620
+ const factory = __addDisposableResource(env_23, await generateStorageFactory(), true);
1649
1621
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1650
1622
  bucket_definitions:
1651
1623
  global:
@@ -1655,49 +1627,51 @@ bucket_definitions:
1655
1627
  storageVersion
1656
1628
  }));
1657
1629
  const bucketStorage = factory.getInstance(syncRules);
1658
- const sourceTable = TEST_TABLE;
1659
- // We simulate two concurrent batches, and nesting is the easiest way to do this.
1630
+ const snapshotWriter = __addDisposableResource(env_23, await bucketStorage.createWriter({
1631
+ ...test_utils.BATCH_OPTIONS,
1632
+ skipExistingRows: true
1633
+ }), true);
1634
+ const streamingWriter = __addDisposableResource(env_23, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1635
+ const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
1636
+ const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
1637
+ // We simulate two concurrent batches; separate writers are enough for this test.
1660
1638
  // For this test, we assume that we start with a row "test1", which is picked up by a snapshot
1661
1639
  // query, right before the delete is streamed. But the snapshot query is only persisted _after_
1662
1640
  // the delete is streamed, and we need to ensure that the streamed delete takes precedence.
1663
- await bucketStorage.startBatch({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true }, async (snapshotBatch) => {
1664
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (streamingBatch) => {
1665
- streamingBatch.save({
1666
- sourceTable,
1667
- tag: storage.SaveOperationTag.DELETE,
1668
- before: {
1669
- id: 'test1'
1670
- },
1671
- beforeReplicaId: test_utils.rid('test1')
1672
- });
1673
- await streamingBatch.commit('2/1');
1674
- await snapshotBatch.save({
1675
- sourceTable,
1676
- tag: storage.SaveOperationTag.INSERT,
1677
- after: {
1678
- id: 'test1',
1679
- description: 'test1a'
1680
- },
1681
- afterReplicaId: test_utils.rid('test1')
1682
- });
1683
- await snapshotBatch.markAllSnapshotDone('3/1');
1684
- await snapshotBatch.commit('1/1');
1685
- await streamingBatch.keepalive('3/1');
1686
- });
1641
+ await streamingWriter.save({
1642
+ sourceTable: streamingTable,
1643
+ tag: storage.SaveOperationTag.DELETE,
1644
+ before: {
1645
+ id: 'test1'
1646
+ },
1647
+ beforeReplicaId: test_utils.rid('test1')
1648
+ });
1649
+ await streamingWriter.commit('2/1');
1650
+ await snapshotWriter.save({
1651
+ sourceTable: snapshotTable,
1652
+ tag: storage.SaveOperationTag.INSERT,
1653
+ after: {
1654
+ id: 'test1',
1655
+ description: 'test1a'
1656
+ },
1657
+ afterReplicaId: test_utils.rid('test1')
1687
1658
  });
1659
+ await snapshotWriter.markAllSnapshotDone('3/1');
1660
+ await snapshotWriter.commit('1/1');
1661
+ await streamingWriter.keepalive('3/1');
1688
1662
  const cp = await bucketStorage.getCheckpoint();
1689
1663
  expect(cp.lsn).toEqual('3/1');
1690
- const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])));
1664
+ const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]));
1691
1665
  expect(data).toEqual([]);
1692
1666
  }
1693
- catch (e_21) {
1694
- env_21.error = e_21;
1695
- env_21.hasError = true;
1667
+ catch (e_23) {
1668
+ env_23.error = e_23;
1669
+ env_23.hasError = true;
1696
1670
  }
1697
1671
  finally {
1698
- const result_21 = __disposeResources(env_21);
1699
- if (result_21)
1700
- await result_21;
1672
+ const result_23 = __disposeResources(env_23);
1673
+ if (result_23)
1674
+ await result_23;
1701
1675
  }
1702
1676
  });
1703
1677
  }
@@ -1709,9 +1683,9 @@ bucket_definitions:
1709
1683
  export function testChecksumBatching(config) {
1710
1684
  const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
1711
1685
  test('checksums for multiple buckets', async () => {
1712
- const env_22 = { stack: [], error: void 0, hasError: false };
1686
+ const env_24 = { stack: [], error: void 0, hasError: false };
1713
1687
  try {
1714
- const factory = __addDisposableResource(env_22, await config.factory(), true);
1688
+ const factory = __addDisposableResource(env_24, await config.factory(), true);
1715
1689
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1716
1690
  bucket_definitions:
1717
1691
  user:
@@ -1722,46 +1696,48 @@ bucket_definitions:
1722
1696
  storageVersion
1723
1697
  }));
1724
1698
  const bucketStorage = factory.getInstance(syncRules);
1725
- const sourceTable = test_utils.makeTestTable('test', ['id'], config);
1726
- await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
1727
- await batch.markAllSnapshotDone('1/1');
1728
- for (let u of ['u1', 'u2', 'u3', 'u4']) {
1729
- for (let t of ['t1', 't2', 't3', 't4']) {
1730
- const id = `${t}_${u}`;
1731
- await batch.save({
1732
- sourceTable,
1733
- tag: storage.SaveOperationTag.INSERT,
1734
- after: {
1735
- id,
1736
- description: `${t} description`,
1737
- user_id: u
1738
- },
1739
- afterReplicaId: test_utils.rid(id)
1740
- });
1741
- }
1699
+ const writer = __addDisposableResource(env_24, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1700
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1701
+ await writer.markAllSnapshotDone('1/1');
1702
+ for (let u of ['u1', 'u2', 'u3', 'u4']) {
1703
+ for (let t of ['t1', 't2', 't3', 't4']) {
1704
+ const id = `${t}_${u}`;
1705
+ await writer.save({
1706
+ sourceTable,
1707
+ tag: storage.SaveOperationTag.INSERT,
1708
+ after: {
1709
+ id,
1710
+ description: `${t} description`,
1711
+ user_id: u
1712
+ },
1713
+ afterReplicaId: test_utils.rid(id)
1714
+ });
1742
1715
  }
1743
- await batch.commit('1/1');
1744
- });
1716
+ }
1717
+ await writer.commit('1/1');
1745
1718
  const { checkpoint } = await bucketStorage.getCheckpoint();
1746
1719
  bucketStorage.clearChecksumCache();
1747
- const buckets = bucketRequests(syncRules, ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]']);
1748
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()];
1720
+ const users = ['u1', 'u2', 'u3', 'u4'];
1721
+ const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
1722
+ const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
1723
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
1749
1724
  checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
1750
- expect(checksums).toEqual([
1751
- { bucket: bucketRequest(syncRules, 'user["u1"]').bucket, count: 4, checksum: 346204588 },
1752
- { bucket: bucketRequest(syncRules, 'user["u2"]').bucket, count: 4, checksum: 5261081 },
1753
- { bucket: bucketRequest(syncRules, 'user["u3"]').bucket, count: 4, checksum: 134760718 },
1754
- { bucket: bucketRequest(syncRules, 'user["u4"]').bucket, count: 4, checksum: -302639724 }
1755
- ]);
1725
+ const expected = bucketRequests.map((request, index) => ({
1726
+ bucket: request.bucket,
1727
+ count: 4,
1728
+ checksum: expectedChecksums[index]
1729
+ }));
1730
+ expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
1731
+ expect(checksums).toEqual(expected);
1756
1732
  }
1757
- catch (e_22) {
1758
- env_22.error = e_22;
1759
- env_22.hasError = true;
1733
+ catch (e_24) {
1734
+ env_24.error = e_24;
1735
+ env_24.hasError = true;
1760
1736
  }
1761
1737
  finally {
1762
- const result_22 = __disposeResources(env_22);
1763
- if (result_22)
1764
- await result_22;
1738
+ const result_24 = __disposeResources(env_24);
1739
+ if (result_24)
1740
+ await result_24;
1765
1741
  }
1766
1742
  });
1767
1743
  }