@powersync/service-core-tests 0.15.0 → 0.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/test-utils/general-utils.d.ts +13 -1
- package/dist/test-utils/general-utils.js +30 -1
- package/dist/test-utils/general-utils.js.map +1 -1
- package/dist/test-utils/stream_utils.js +2 -2
- package/dist/test-utils/stream_utils.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +266 -257
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +36 -57
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +839 -863
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +228 -236
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +81 -89
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +468 -462
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/package.json +3 -3
- package/src/test-utils/general-utils.ts +41 -2
- package/src/test-utils/stream_utils.ts +2 -2
- package/src/tests/register-compacting-tests.ts +279 -270
- package/src/tests/register-data-storage-checkpoint-tests.ts +36 -57
- package/src/tests/register-data-storage-data-tests.ts +673 -770
- package/src/tests/register-data-storage-parameter-tests.ts +245 -257
- package/src/tests/register-parameter-compacting-tests.ts +84 -92
- package/src/tests/register-sync-tests.ts +375 -391
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -54,7 +54,6 @@ import { CURRENT_STORAGE_VERSION, getUuidReplicaIdentityBson, reduceBucket, stor
|
|
|
54
54
|
import { describe, expect, test } from 'vitest';
|
|
55
55
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
56
56
|
import { bucketRequest } from '../test-utils/test-utils-index.js';
|
|
57
|
-
import { bucketRequestMap, bucketRequests } from './util.js';
|
|
58
57
|
/**
|
|
59
58
|
* Normalize data from OplogEntries for comparison in tests.
|
|
60
59
|
* Tests typically expect the stringified result
|
|
@@ -78,7 +77,6 @@ const normalizeOplogData = (data) => {
|
|
|
78
77
|
export function registerDataStorageDataTests(config) {
|
|
79
78
|
const generateStorageFactory = config.factory;
|
|
80
79
|
const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
|
|
81
|
-
const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
|
|
82
80
|
test('removing row', async () => {
|
|
83
81
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
84
82
|
try {
|
|
@@ -90,27 +88,27 @@ bucket_definitions:
|
|
|
90
88
|
- SELECT id, description FROM "%"
|
|
91
89
|
`, { storageVersion }));
|
|
92
90
|
const bucketStorage = factory.getInstance(syncRules);
|
|
93
|
-
await bucketStorage.
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
});
|
|
105
|
-
await batch.save({
|
|
106
|
-
sourceTable,
|
|
107
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
108
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
109
|
-
});
|
|
110
|
-
await batch.commit('1/1');
|
|
91
|
+
const writer = __addDisposableResource(env_1, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
92
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
93
|
+
await writer.markAllSnapshotDone('1/1');
|
|
94
|
+
await writer.save({
|
|
95
|
+
sourceTable: testTable,
|
|
96
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
97
|
+
after: {
|
|
98
|
+
id: 'test1',
|
|
99
|
+
description: 'test1'
|
|
100
|
+
},
|
|
101
|
+
afterReplicaId: test_utils.rid('test1')
|
|
111
102
|
});
|
|
103
|
+
await writer.save({
|
|
104
|
+
sourceTable: testTable,
|
|
105
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
106
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
107
|
+
});
|
|
108
|
+
await writer.commit('1/1');
|
|
112
109
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
113
|
-
const
|
|
110
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
111
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
114
112
|
const data = batch[0].chunkData.data.map((d) => {
|
|
115
113
|
return {
|
|
116
114
|
op: d.op,
|
|
@@ -124,12 +122,10 @@ bucket_definitions:
|
|
|
124
122
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
125
123
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
126
124
|
]);
|
|
127
|
-
const checksums = [
|
|
128
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
129
|
-
];
|
|
125
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
130
126
|
expect(checksums).toEqual([
|
|
131
127
|
{
|
|
132
|
-
bucket:
|
|
128
|
+
bucket: request.bucket,
|
|
133
129
|
checksum: (c1 + c2) & 0xffffffff,
|
|
134
130
|
count: 2
|
|
135
131
|
}
|
|
@@ -156,31 +152,28 @@ bucket_definitions:
|
|
|
156
152
|
- SELECT id, description FROM "%"
|
|
157
153
|
`, { storageVersion }));
|
|
158
154
|
const bucketStorage = factory.getInstance(syncRules);
|
|
159
|
-
const
|
|
160
|
-
await
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
});
|
|
167
|
-
await batch.commit('0/1');
|
|
155
|
+
const writer = __addDisposableResource(env_2, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
156
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
157
|
+
await writer.markAllSnapshotDone('1/1');
|
|
158
|
+
await writer.save({
|
|
159
|
+
sourceTable: testTable,
|
|
160
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
161
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
168
162
|
});
|
|
169
|
-
await
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
afterReplicaId: test_utils.rid('test1')
|
|
179
|
-
});
|
|
180
|
-
await batch.commit('2/1');
|
|
163
|
+
await writer.commit('0/1');
|
|
164
|
+
await writer.save({
|
|
165
|
+
sourceTable: testTable,
|
|
166
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
167
|
+
after: {
|
|
168
|
+
id: 'test1',
|
|
169
|
+
description: 'test1'
|
|
170
|
+
},
|
|
171
|
+
afterReplicaId: test_utils.rid('test1')
|
|
181
172
|
});
|
|
173
|
+
await writer.commit('2/1');
|
|
182
174
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
183
|
-
const
|
|
175
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
176
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
184
177
|
const data = batch[0].chunkData.data.map((d) => {
|
|
185
178
|
return {
|
|
186
179
|
op: d.op,
|
|
@@ -190,12 +183,10 @@ bucket_definitions:
|
|
|
190
183
|
});
|
|
191
184
|
const c1 = 2871785649;
|
|
192
185
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
193
|
-
const checksums = [
|
|
194
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
195
|
-
];
|
|
186
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
196
187
|
expect(checksums).toEqual([
|
|
197
188
|
{
|
|
198
|
-
bucket:
|
|
189
|
+
bucket: request.bucket,
|
|
199
190
|
checksum: c1 & 0xffffffff,
|
|
200
191
|
count: 1
|
|
201
192
|
}
|
|
@@ -223,35 +214,32 @@ bucket_definitions:
|
|
|
223
214
|
- SELECT id, description FROM "%"
|
|
224
215
|
`, { storageVersion }));
|
|
225
216
|
const bucketStorage = factory.getInstance(syncRules);
|
|
226
|
-
const
|
|
227
|
-
await
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
});
|
|
234
|
-
await batch.commit('0/1');
|
|
217
|
+
const writer = __addDisposableResource(env_3, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
218
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
219
|
+
await writer.markAllSnapshotDone('1/1');
|
|
220
|
+
await writer.save({
|
|
221
|
+
sourceTable: testTable,
|
|
222
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
223
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
235
224
|
});
|
|
236
|
-
await
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
afterReplicaId: test_utils.rid('test1')
|
|
250
|
-
});
|
|
251
|
-
await batch.commit('2/1');
|
|
225
|
+
await writer.commit('0/1');
|
|
226
|
+
await writer.save({
|
|
227
|
+
sourceTable: testTable,
|
|
228
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
229
|
+
before: {
|
|
230
|
+
id: 'test1'
|
|
231
|
+
},
|
|
232
|
+
after: {
|
|
233
|
+
id: 'test1',
|
|
234
|
+
description: 'test1'
|
|
235
|
+
},
|
|
236
|
+
beforeReplicaId: test_utils.rid('test1'),
|
|
237
|
+
afterReplicaId: test_utils.rid('test1')
|
|
252
238
|
});
|
|
239
|
+
await writer.commit('2/1');
|
|
253
240
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
254
|
-
const
|
|
241
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
242
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
255
243
|
const data = batch[0].chunkData.data.map((d) => {
|
|
256
244
|
return {
|
|
257
245
|
op: d.op,
|
|
@@ -261,12 +249,10 @@ bucket_definitions:
|
|
|
261
249
|
});
|
|
262
250
|
const c1 = 2871785649;
|
|
263
251
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
264
|
-
const checksums = [
|
|
265
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
266
|
-
];
|
|
252
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
267
253
|
expect(checksums).toEqual([
|
|
268
254
|
{
|
|
269
|
-
bucket:
|
|
255
|
+
bucket: request.bucket,
|
|
270
256
|
checksum: c1 & 0xffffffff,
|
|
271
257
|
count: 1
|
|
272
258
|
}
|
|
@@ -295,27 +281,27 @@ bucket_definitions:
|
|
|
295
281
|
storageVersion
|
|
296
282
|
}));
|
|
297
283
|
const bucketStorage = factory.getInstance(syncRules);
|
|
298
|
-
await bucketStorage.
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
});
|
|
315
|
-
await batch.commit('1/1');
|
|
284
|
+
const writer = __addDisposableResource(env_4, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
285
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
286
|
+
await writer.markAllSnapshotDone('1/1');
|
|
287
|
+
await writer.save({
|
|
288
|
+
sourceTable: testTable,
|
|
289
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
290
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
291
|
+
});
|
|
292
|
+
await writer.save({
|
|
293
|
+
sourceTable: testTable,
|
|
294
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
295
|
+
after: {
|
|
296
|
+
id: 'test1',
|
|
297
|
+
description: 'test1'
|
|
298
|
+
},
|
|
299
|
+
afterReplicaId: test_utils.rid('test1')
|
|
316
300
|
});
|
|
301
|
+
await writer.commit('1/1');
|
|
317
302
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
318
|
-
const
|
|
303
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
304
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
319
305
|
const data = batch[0].chunkData.data.map((d) => {
|
|
320
306
|
return {
|
|
321
307
|
op: d.op,
|
|
@@ -325,12 +311,10 @@ bucket_definitions:
|
|
|
325
311
|
});
|
|
326
312
|
const c1 = 2871785649;
|
|
327
313
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
328
|
-
const checksums = [
|
|
329
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
330
|
-
];
|
|
314
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
331
315
|
expect(checksums).toEqual([
|
|
332
316
|
{
|
|
333
|
-
bucket:
|
|
317
|
+
bucket: request.bucket,
|
|
334
318
|
checksum: c1 & 0xffffffff,
|
|
335
319
|
count: 1
|
|
336
320
|
}
|
|
@@ -359,46 +343,73 @@ bucket_definitions:
|
|
|
359
343
|
storageVersion
|
|
360
344
|
}));
|
|
361
345
|
const bucketStorage = factory.getInstance(syncRules);
|
|
362
|
-
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
sourceTable,
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
346
|
+
{
|
|
347
|
+
const env_6 = { stack: [], error: void 0, hasError: false };
|
|
348
|
+
try {
|
|
349
|
+
const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
350
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
351
|
+
await writer.markAllSnapshotDone('1/1');
|
|
352
|
+
await writer.save({
|
|
353
|
+
sourceTable,
|
|
354
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
355
|
+
after: {
|
|
356
|
+
id: 'test1',
|
|
357
|
+
description: 'test1'
|
|
358
|
+
},
|
|
359
|
+
afterReplicaId: test_utils.rid('test1')
|
|
360
|
+
});
|
|
361
|
+
await writer.save({
|
|
362
|
+
sourceTable,
|
|
363
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
364
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
365
|
+
});
|
|
366
|
+
await writer.save({
|
|
367
|
+
sourceTable,
|
|
368
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
369
|
+
after: {
|
|
370
|
+
id: 'test1',
|
|
371
|
+
description: 'test1'
|
|
372
|
+
},
|
|
373
|
+
afterReplicaId: test_utils.rid('test1')
|
|
374
|
+
});
|
|
375
|
+
await writer.commit('1/1');
|
|
376
|
+
}
|
|
377
|
+
catch (e_5) {
|
|
378
|
+
env_6.error = e_5;
|
|
379
|
+
env_6.hasError = true;
|
|
380
|
+
}
|
|
381
|
+
finally {
|
|
382
|
+
const result_5 = __disposeResources(env_6);
|
|
383
|
+
if (result_5)
|
|
384
|
+
await result_5;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
{
|
|
388
|
+
const env_7 = { stack: [], error: void 0, hasError: false };
|
|
389
|
+
try {
|
|
390
|
+
const writer = __addDisposableResource(env_7, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
391
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
392
|
+
await writer.markAllSnapshotDone('1/1');
|
|
393
|
+
await writer.save({
|
|
394
|
+
sourceTable,
|
|
395
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
396
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
397
|
+
});
|
|
398
|
+
await writer.commit('2/1');
|
|
399
|
+
}
|
|
400
|
+
catch (e_6) {
|
|
401
|
+
env_7.error = e_6;
|
|
402
|
+
env_7.hasError = true;
|
|
403
|
+
}
|
|
404
|
+
finally {
|
|
405
|
+
const result_6 = __disposeResources(env_7);
|
|
406
|
+
if (result_6)
|
|
407
|
+
await result_6;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
400
410
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
401
|
-
const
|
|
411
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
412
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
402
413
|
expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
|
|
403
414
|
const data = batch[0].chunkData.data.map((d) => {
|
|
404
415
|
return {
|
|
@@ -409,20 +420,20 @@ bucket_definitions:
|
|
|
409
420
|
});
|
|
410
421
|
expect(data).toMatchSnapshot();
|
|
411
422
|
}
|
|
412
|
-
catch (
|
|
413
|
-
env_5.error =
|
|
423
|
+
catch (e_7) {
|
|
424
|
+
env_5.error = e_7;
|
|
414
425
|
env_5.hasError = true;
|
|
415
426
|
}
|
|
416
427
|
finally {
|
|
417
|
-
const
|
|
418
|
-
if (
|
|
419
|
-
await
|
|
428
|
+
const result_7 = __disposeResources(env_5);
|
|
429
|
+
if (result_7)
|
|
430
|
+
await result_7;
|
|
420
431
|
}
|
|
421
432
|
});
|
|
422
433
|
test('changing client ids', async () => {
|
|
423
|
-
const
|
|
434
|
+
const env_8 = { stack: [], error: void 0, hasError: false };
|
|
424
435
|
try {
|
|
425
|
-
const factory = __addDisposableResource(
|
|
436
|
+
const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
|
|
426
437
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
427
438
|
bucket_definitions:
|
|
428
439
|
global:
|
|
@@ -432,43 +443,42 @@ bucket_definitions:
|
|
|
432
443
|
storageVersion
|
|
433
444
|
}));
|
|
434
445
|
const bucketStorage = factory.getInstance(syncRules);
|
|
435
|
-
const
|
|
436
|
-
await
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
});
|
|
458
|
-
await batch.save({
|
|
459
|
-
sourceTable,
|
|
460
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
461
|
-
after: {
|
|
462
|
-
id: 'test2',
|
|
463
|
-
client_id: 'client2',
|
|
464
|
-
description: 'test2'
|
|
465
|
-
},
|
|
466
|
-
afterReplicaId: test_utils.rid('test2')
|
|
467
|
-
});
|
|
468
|
-
await batch.commit('1/1');
|
|
446
|
+
const writer = __addDisposableResource(env_8, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
447
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
448
|
+
await writer.markAllSnapshotDone('1/1');
|
|
449
|
+
await writer.save({
|
|
450
|
+
sourceTable,
|
|
451
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
452
|
+
after: {
|
|
453
|
+
id: 'test1',
|
|
454
|
+
client_id: 'client1a',
|
|
455
|
+
description: 'test1a'
|
|
456
|
+
},
|
|
457
|
+
afterReplicaId: test_utils.rid('test1')
|
|
458
|
+
});
|
|
459
|
+
await writer.save({
|
|
460
|
+
sourceTable,
|
|
461
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
462
|
+
after: {
|
|
463
|
+
id: 'test1',
|
|
464
|
+
client_id: 'client1b',
|
|
465
|
+
description: 'test1b'
|
|
466
|
+
},
|
|
467
|
+
afterReplicaId: test_utils.rid('test1')
|
|
469
468
|
});
|
|
469
|
+
await writer.save({
|
|
470
|
+
sourceTable,
|
|
471
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
472
|
+
after: {
|
|
473
|
+
id: 'test2',
|
|
474
|
+
client_id: 'client2',
|
|
475
|
+
description: 'test2'
|
|
476
|
+
},
|
|
477
|
+
afterReplicaId: test_utils.rid('test2')
|
|
478
|
+
});
|
|
479
|
+
await writer.commit('1/1');
|
|
470
480
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
471
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
481
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
|
|
472
482
|
const data = batch[0].chunkData.data.map((d) => {
|
|
473
483
|
return {
|
|
474
484
|
op: d.op,
|
|
@@ -482,20 +492,20 @@ bucket_definitions:
|
|
|
482
492
|
{ op: 'PUT', object_id: 'client2' }
|
|
483
493
|
]);
|
|
484
494
|
}
|
|
485
|
-
catch (
|
|
486
|
-
|
|
487
|
-
|
|
495
|
+
catch (e_8) {
|
|
496
|
+
env_8.error = e_8;
|
|
497
|
+
env_8.hasError = true;
|
|
488
498
|
}
|
|
489
499
|
finally {
|
|
490
|
-
const
|
|
491
|
-
if (
|
|
492
|
-
await
|
|
500
|
+
const result_8 = __disposeResources(env_8);
|
|
501
|
+
if (result_8)
|
|
502
|
+
await result_8;
|
|
493
503
|
}
|
|
494
504
|
});
|
|
495
505
|
test('re-apply delete', async () => {
|
|
496
|
-
const
|
|
506
|
+
const env_9 = { stack: [], error: void 0, hasError: false };
|
|
497
507
|
try {
|
|
498
|
-
const factory = __addDisposableResource(
|
|
508
|
+
const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
|
|
499
509
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
500
510
|
bucket_definitions:
|
|
501
511
|
global:
|
|
@@ -505,38 +515,34 @@ bucket_definitions:
|
|
|
505
515
|
storageVersion
|
|
506
516
|
}));
|
|
507
517
|
const bucketStorage = factory.getInstance(syncRules);
|
|
508
|
-
await bucketStorage.
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
});
|
|
518
|
+
const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
519
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
520
|
+
await writer.markAllSnapshotDone('1/1');
|
|
521
|
+
await writer.save({
|
|
522
|
+
sourceTable,
|
|
523
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
524
|
+
after: {
|
|
525
|
+
id: 'test1',
|
|
526
|
+
description: 'test1'
|
|
527
|
+
},
|
|
528
|
+
afterReplicaId: test_utils.rid('test1')
|
|
520
529
|
});
|
|
521
|
-
await
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
527
|
-
});
|
|
528
|
-
await batch.commit('1/1');
|
|
530
|
+
await writer.flush();
|
|
531
|
+
await writer.save({
|
|
532
|
+
sourceTable,
|
|
533
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
534
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
529
535
|
});
|
|
530
|
-
await
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
536
|
-
});
|
|
536
|
+
await writer.commit('1/1');
|
|
537
|
+
await writer.save({
|
|
538
|
+
sourceTable,
|
|
539
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
540
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
537
541
|
});
|
|
542
|
+
await writer.flush();
|
|
538
543
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
539
|
-
const
|
|
544
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
545
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
540
546
|
const data = batch[0].chunkData.data.map((d) => {
|
|
541
547
|
return {
|
|
542
548
|
op: d.op,
|
|
@@ -550,9 +556,7 @@ bucket_definitions:
|
|
|
550
556
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
551
557
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
552
558
|
]);
|
|
553
|
-
const checksums = [
|
|
554
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
555
|
-
];
|
|
559
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
556
560
|
expect(checksums).toEqual([
|
|
557
561
|
{
|
|
558
562
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -561,20 +565,20 @@ bucket_definitions:
|
|
|
561
565
|
}
|
|
562
566
|
]);
|
|
563
567
|
}
|
|
564
|
-
catch (
|
|
565
|
-
|
|
566
|
-
|
|
568
|
+
catch (e_9) {
|
|
569
|
+
env_9.error = e_9;
|
|
570
|
+
env_9.hasError = true;
|
|
567
571
|
}
|
|
568
572
|
finally {
|
|
569
|
-
const
|
|
570
|
-
if (
|
|
571
|
-
await
|
|
573
|
+
const result_9 = __disposeResources(env_9);
|
|
574
|
+
if (result_9)
|
|
575
|
+
await result_9;
|
|
572
576
|
}
|
|
573
577
|
});
|
|
574
578
|
test('re-apply update + delete', async () => {
|
|
575
|
-
const
|
|
579
|
+
const env_10 = { stack: [], error: void 0, hasError: false };
|
|
576
580
|
try {
|
|
577
|
-
const factory = __addDisposableResource(
|
|
581
|
+
const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
|
|
578
582
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
579
583
|
bucket_definitions:
|
|
580
584
|
global:
|
|
@@ -582,77 +586,72 @@ bucket_definitions:
|
|
|
582
586
|
- SELECT id, description FROM "%"
|
|
583
587
|
`, { storageVersion }));
|
|
584
588
|
const bucketStorage = factory.getInstance(syncRules);
|
|
585
|
-
await bucketStorage.
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
});
|
|
589
|
+
const writer = __addDisposableResource(env_10, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
590
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
591
|
+
await writer.markAllSnapshotDone('1/1');
|
|
592
|
+
await writer.save({
|
|
593
|
+
sourceTable,
|
|
594
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
595
|
+
after: {
|
|
596
|
+
id: 'test1',
|
|
597
|
+
description: 'test1'
|
|
598
|
+
},
|
|
599
|
+
afterReplicaId: test_utils.rid('test1')
|
|
597
600
|
});
|
|
598
|
-
await
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
afterReplicaId: test_utils.rid('test1')
|
|
609
|
-
});
|
|
610
|
-
await batch.save({
|
|
611
|
-
sourceTable,
|
|
612
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
613
|
-
after: {
|
|
614
|
-
id: 'test1',
|
|
615
|
-
description: undefined
|
|
616
|
-
},
|
|
617
|
-
afterReplicaId: test_utils.rid('test1')
|
|
618
|
-
});
|
|
619
|
-
await batch.save({
|
|
620
|
-
sourceTable,
|
|
621
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
622
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
623
|
-
});
|
|
624
|
-
await batch.commit('1/1');
|
|
601
|
+
await writer.flush();
|
|
602
|
+
await writer.markAllSnapshotDone('1/1');
|
|
603
|
+
await writer.save({
|
|
604
|
+
sourceTable,
|
|
605
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
606
|
+
after: {
|
|
607
|
+
id: 'test1',
|
|
608
|
+
description: undefined
|
|
609
|
+
},
|
|
610
|
+
afterReplicaId: test_utils.rid('test1')
|
|
625
611
|
});
|
|
626
|
-
await
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
651
|
-
});
|
|
652
|
-
await batch.commit('2/1');
|
|
612
|
+
await writer.save({
|
|
613
|
+
sourceTable,
|
|
614
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
615
|
+
after: {
|
|
616
|
+
id: 'test1',
|
|
617
|
+
description: undefined
|
|
618
|
+
},
|
|
619
|
+
afterReplicaId: test_utils.rid('test1')
|
|
620
|
+
});
|
|
621
|
+
await writer.save({
|
|
622
|
+
sourceTable,
|
|
623
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
624
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
625
|
+
});
|
|
626
|
+
await writer.commit('1/1');
|
|
627
|
+
await writer.markAllSnapshotDone('1/1');
|
|
628
|
+
await writer.save({
|
|
629
|
+
sourceTable,
|
|
630
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
631
|
+
after: {
|
|
632
|
+
id: 'test1',
|
|
633
|
+
description: undefined
|
|
634
|
+
},
|
|
635
|
+
afterReplicaId: test_utils.rid('test1')
|
|
653
636
|
});
|
|
637
|
+
await writer.save({
|
|
638
|
+
sourceTable,
|
|
639
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
640
|
+
after: {
|
|
641
|
+
id: 'test1',
|
|
642
|
+
description: undefined
|
|
643
|
+
},
|
|
644
|
+
afterReplicaId: test_utils.rid('test1')
|
|
645
|
+
});
|
|
646
|
+
await writer.save({
|
|
647
|
+
sourceTable,
|
|
648
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
649
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
650
|
+
});
|
|
651
|
+
await writer.commit('2/1');
|
|
654
652
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
655
|
-
const
|
|
653
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
654
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
656
655
|
const data = batch[0].chunkData.data.map((d) => {
|
|
657
656
|
return {
|
|
658
657
|
op: d.op,
|
|
@@ -668,9 +667,7 @@ bucket_definitions:
|
|
|
668
667
|
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
669
668
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
670
669
|
]);
|
|
671
|
-
const checksums = [
|
|
672
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
673
|
-
];
|
|
670
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
674
671
|
expect(checksums).toEqual([
|
|
675
672
|
{
|
|
676
673
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -679,18 +676,18 @@ bucket_definitions:
|
|
|
679
676
|
}
|
|
680
677
|
]);
|
|
681
678
|
}
|
|
682
|
-
catch (
|
|
683
|
-
|
|
684
|
-
|
|
679
|
+
catch (e_10) {
|
|
680
|
+
env_10.error = e_10;
|
|
681
|
+
env_10.hasError = true;
|
|
685
682
|
}
|
|
686
683
|
finally {
|
|
687
|
-
const
|
|
688
|
-
if (
|
|
689
|
-
await
|
|
684
|
+
const result_10 = __disposeResources(env_10);
|
|
685
|
+
if (result_10)
|
|
686
|
+
await result_10;
|
|
690
687
|
}
|
|
691
688
|
});
|
|
692
689
|
test('batch with overlapping replica ids', async () => {
|
|
693
|
-
const
|
|
690
|
+
const env_11 = { stack: [], error: void 0, hasError: false };
|
|
694
691
|
try {
|
|
695
692
|
// This test checks that we get the correct output when processing rows with:
|
|
696
693
|
// 1. changing replica ids
|
|
@@ -699,7 +696,7 @@ bucket_definitions:
|
|
|
699
696
|
// It can break at two places:
|
|
700
697
|
// 1. Not getting the correct "current_data" state for each operation.
|
|
701
698
|
// 2. Output order not being correct.
|
|
702
|
-
const factory = __addDisposableResource(
|
|
699
|
+
const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
|
|
703
700
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
704
701
|
bucket_definitions:
|
|
705
702
|
global:
|
|
@@ -707,105 +704,104 @@ bucket_definitions:
|
|
|
707
704
|
- SELECT id, description FROM "test"
|
|
708
705
|
`, { storageVersion }));
|
|
709
706
|
const bucketStorage = factory.getInstance(syncRules);
|
|
707
|
+
const writer = __addDisposableResource(env_11, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
708
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
710
709
|
// Pre-setup
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
},
|
|
721
|
-
afterReplicaId: test_utils.rid('test1')
|
|
722
|
-
});
|
|
723
|
-
await batch.save({
|
|
724
|
-
sourceTable,
|
|
725
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
726
|
-
after: {
|
|
727
|
-
id: 'test2',
|
|
728
|
-
description: 'test2a'
|
|
729
|
-
},
|
|
730
|
-
afterReplicaId: test_utils.rid('test2')
|
|
731
|
-
});
|
|
710
|
+
await writer.markAllSnapshotDone('1/1');
|
|
711
|
+
await writer.save({
|
|
712
|
+
sourceTable,
|
|
713
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
714
|
+
after: {
|
|
715
|
+
id: 'test1',
|
|
716
|
+
description: 'test1a'
|
|
717
|
+
},
|
|
718
|
+
afterReplicaId: test_utils.rid('test1')
|
|
732
719
|
});
|
|
720
|
+
await writer.save({
|
|
721
|
+
sourceTable,
|
|
722
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
723
|
+
after: {
|
|
724
|
+
id: 'test2',
|
|
725
|
+
description: 'test2a'
|
|
726
|
+
},
|
|
727
|
+
afterReplicaId: test_utils.rid('test2')
|
|
728
|
+
});
|
|
729
|
+
const result1 = await writer.flush();
|
|
733
730
|
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
734
731
|
// Test batch
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
},
|
|
791
|
-
afterReplicaId: test_utils.rid('test4')
|
|
792
|
-
});
|
|
793
|
-
await batch.save({
|
|
794
|
-
sourceTable,
|
|
795
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
796
|
-
before: {
|
|
797
|
-
id: 'test4'
|
|
798
|
-
},
|
|
799
|
-
beforeReplicaId: test_utils.rid('test4'),
|
|
800
|
-
after: {
|
|
801
|
-
id: 'test5',
|
|
802
|
-
description: 'test5d'
|
|
803
|
-
},
|
|
804
|
-
afterReplicaId: test_utils.rid('test5')
|
|
805
|
-
});
|
|
732
|
+
// b
|
|
733
|
+
await writer.save({
|
|
734
|
+
sourceTable,
|
|
735
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
736
|
+
after: {
|
|
737
|
+
id: 'test1',
|
|
738
|
+
description: 'test1b'
|
|
739
|
+
},
|
|
740
|
+
afterReplicaId: test_utils.rid('test1')
|
|
741
|
+
});
|
|
742
|
+
await writer.save({
|
|
743
|
+
sourceTable,
|
|
744
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
745
|
+
before: {
|
|
746
|
+
id: 'test1'
|
|
747
|
+
},
|
|
748
|
+
beforeReplicaId: test_utils.rid('test1'),
|
|
749
|
+
after: {
|
|
750
|
+
id: 'test2',
|
|
751
|
+
description: 'test2b'
|
|
752
|
+
},
|
|
753
|
+
afterReplicaId: test_utils.rid('test2')
|
|
754
|
+
});
|
|
755
|
+
await writer.save({
|
|
756
|
+
sourceTable,
|
|
757
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
758
|
+
before: {
|
|
759
|
+
id: 'test2'
|
|
760
|
+
},
|
|
761
|
+
beforeReplicaId: test_utils.rid('test2'),
|
|
762
|
+
after: {
|
|
763
|
+
id: 'test3',
|
|
764
|
+
description: 'test3b'
|
|
765
|
+
},
|
|
766
|
+
afterReplicaId: test_utils.rid('test3')
|
|
767
|
+
});
|
|
768
|
+
// c
|
|
769
|
+
await writer.save({
|
|
770
|
+
sourceTable,
|
|
771
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
772
|
+
after: {
|
|
773
|
+
id: 'test2',
|
|
774
|
+
description: 'test2c'
|
|
775
|
+
},
|
|
776
|
+
afterReplicaId: test_utils.rid('test2')
|
|
777
|
+
});
|
|
778
|
+
// d
|
|
779
|
+
await writer.save({
|
|
780
|
+
sourceTable,
|
|
781
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
782
|
+
after: {
|
|
783
|
+
id: 'test4',
|
|
784
|
+
description: 'test4d'
|
|
785
|
+
},
|
|
786
|
+
afterReplicaId: test_utils.rid('test4')
|
|
806
787
|
});
|
|
788
|
+
await writer.save({
|
|
789
|
+
sourceTable,
|
|
790
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
791
|
+
before: {
|
|
792
|
+
id: 'test4'
|
|
793
|
+
},
|
|
794
|
+
beforeReplicaId: test_utils.rid('test4'),
|
|
795
|
+
after: {
|
|
796
|
+
id: 'test5',
|
|
797
|
+
description: 'test5d'
|
|
798
|
+
},
|
|
799
|
+
afterReplicaId: test_utils.rid('test5')
|
|
800
|
+
});
|
|
801
|
+
const result2 = await writer.flush();
|
|
807
802
|
const checkpoint2 = result2.flushed_op;
|
|
808
|
-
const
|
|
803
|
+
const request = bucketRequest(syncRules, 'global[]', checkpoint1);
|
|
804
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
|
|
809
805
|
const data = batch[0].chunkData.data.map((d) => {
|
|
810
806
|
return {
|
|
811
807
|
op: d.op,
|
|
@@ -829,18 +825,18 @@ bucket_definitions:
|
|
|
829
825
|
{ op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
|
|
830
826
|
]);
|
|
831
827
|
}
|
|
832
|
-
catch (
|
|
833
|
-
|
|
834
|
-
|
|
828
|
+
catch (e_11) {
|
|
829
|
+
env_11.error = e_11;
|
|
830
|
+
env_11.hasError = true;
|
|
835
831
|
}
|
|
836
832
|
finally {
|
|
837
|
-
const
|
|
838
|
-
if (
|
|
839
|
-
await
|
|
833
|
+
const result_11 = __disposeResources(env_11);
|
|
834
|
+
if (result_11)
|
|
835
|
+
await result_11;
|
|
840
836
|
}
|
|
841
837
|
});
|
|
842
838
|
test('changed data with replica identity full', async () => {
|
|
843
|
-
const
|
|
839
|
+
const env_12 = { stack: [], error: void 0, hasError: false };
|
|
844
840
|
try {
|
|
845
841
|
function rid2(id, description) {
|
|
846
842
|
return getUuidReplicaIdentityBson({ id, description }, [
|
|
@@ -848,7 +844,7 @@ bucket_definitions:
|
|
|
848
844
|
{ name: 'description', type: 'VARCHAR', typeId: 25 }
|
|
849
845
|
]);
|
|
850
846
|
}
|
|
851
|
-
const factory = __addDisposableResource(
|
|
847
|
+
const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
|
|
852
848
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
853
849
|
bucket_definitions:
|
|
854
850
|
global:
|
|
@@ -858,53 +854,52 @@ bucket_definitions:
|
|
|
858
854
|
storageVersion
|
|
859
855
|
}));
|
|
860
856
|
const bucketStorage = factory.getInstance(syncRules);
|
|
857
|
+
const writer = __addDisposableResource(env_12, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
861
858
|
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
862
859
|
// Pre-setup
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
afterReplicaId: rid2('test1', 'test1a')
|
|
873
|
-
});
|
|
860
|
+
await writer.markAllSnapshotDone('1/1');
|
|
861
|
+
await writer.save({
|
|
862
|
+
sourceTable,
|
|
863
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
864
|
+
after: {
|
|
865
|
+
id: 'test1',
|
|
866
|
+
description: 'test1a'
|
|
867
|
+
},
|
|
868
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
874
869
|
});
|
|
870
|
+
const result1 = await writer.flush();
|
|
875
871
|
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
afterReplicaId: rid2('test1', 'test1b')
|
|
891
|
-
});
|
|
872
|
+
// Unchanged, but has a before id
|
|
873
|
+
await writer.save({
|
|
874
|
+
sourceTable,
|
|
875
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
876
|
+
before: {
|
|
877
|
+
id: 'test1',
|
|
878
|
+
description: 'test1a'
|
|
879
|
+
},
|
|
880
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
881
|
+
after: {
|
|
882
|
+
id: 'test1',
|
|
883
|
+
description: 'test1b'
|
|
884
|
+
},
|
|
885
|
+
afterReplicaId: rid2('test1', 'test1b')
|
|
892
886
|
});
|
|
893
|
-
const
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
});
|
|
887
|
+
const result2 = await writer.flush();
|
|
888
|
+
// Delete
|
|
889
|
+
await writer.save({
|
|
890
|
+
sourceTable,
|
|
891
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
892
|
+
before: {
|
|
893
|
+
id: 'test1',
|
|
894
|
+
description: 'test1b'
|
|
895
|
+
},
|
|
896
|
+
beforeReplicaId: rid2('test1', 'test1b'),
|
|
897
|
+
after: undefined
|
|
905
898
|
});
|
|
899
|
+
const result3 = await writer.flush();
|
|
906
900
|
const checkpoint3 = result3.flushed_op;
|
|
907
|
-
const
|
|
901
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
902
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
|
|
908
903
|
const data = batch[0].chunkData.data.map((d) => {
|
|
909
904
|
return {
|
|
910
905
|
op: d.op,
|
|
@@ -938,18 +933,18 @@ bucket_definitions:
|
|
|
938
933
|
}
|
|
939
934
|
]);
|
|
940
935
|
}
|
|
941
|
-
catch (
|
|
942
|
-
|
|
943
|
-
|
|
936
|
+
catch (e_12) {
|
|
937
|
+
env_12.error = e_12;
|
|
938
|
+
env_12.hasError = true;
|
|
944
939
|
}
|
|
945
940
|
finally {
|
|
946
|
-
const
|
|
947
|
-
if (
|
|
948
|
-
await
|
|
941
|
+
const result_12 = __disposeResources(env_12);
|
|
942
|
+
if (result_12)
|
|
943
|
+
await result_12;
|
|
949
944
|
}
|
|
950
945
|
});
|
|
951
946
|
test('unchanged data with replica identity full', async () => {
|
|
952
|
-
const
|
|
947
|
+
const env_13 = { stack: [], error: void 0, hasError: false };
|
|
953
948
|
try {
|
|
954
949
|
function rid2(id, description) {
|
|
955
950
|
return getUuidReplicaIdentityBson({ id, description }, [
|
|
@@ -957,7 +952,7 @@ bucket_definitions:
|
|
|
957
952
|
{ name: 'description', type: 'VARCHAR', typeId: 25 }
|
|
958
953
|
]);
|
|
959
954
|
}
|
|
960
|
-
const factory = __addDisposableResource(
|
|
955
|
+
const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
|
|
961
956
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
962
957
|
bucket_definitions:
|
|
963
958
|
global:
|
|
@@ -967,53 +962,52 @@ bucket_definitions:
|
|
|
967
962
|
storageVersion
|
|
968
963
|
}));
|
|
969
964
|
const bucketStorage = factory.getInstance(syncRules);
|
|
965
|
+
const writer = __addDisposableResource(env_13, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
970
966
|
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
971
967
|
// Pre-setup
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
afterReplicaId: rid2('test1', 'test1a')
|
|
982
|
-
});
|
|
968
|
+
await writer.markAllSnapshotDone('1/1');
|
|
969
|
+
await writer.save({
|
|
970
|
+
sourceTable,
|
|
971
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
972
|
+
after: {
|
|
973
|
+
id: 'test1',
|
|
974
|
+
description: 'test1a'
|
|
975
|
+
},
|
|
976
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
983
977
|
});
|
|
978
|
+
const result1 = await writer.flush();
|
|
984
979
|
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
beforeReplicaId: rid2('test1', 'test1a'),
|
|
1012
|
-
after: undefined
|
|
1013
|
-
});
|
|
980
|
+
// Unchanged, but has a before id
|
|
981
|
+
await writer.save({
|
|
982
|
+
sourceTable,
|
|
983
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
984
|
+
before: {
|
|
985
|
+
id: 'test1',
|
|
986
|
+
description: 'test1a'
|
|
987
|
+
},
|
|
988
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
989
|
+
after: {
|
|
990
|
+
id: 'test1',
|
|
991
|
+
description: 'test1a'
|
|
992
|
+
},
|
|
993
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
994
|
+
});
|
|
995
|
+
const result2 = await writer.flush();
|
|
996
|
+
// Delete
|
|
997
|
+
await writer.save({
|
|
998
|
+
sourceTable,
|
|
999
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
1000
|
+
before: {
|
|
1001
|
+
id: 'test1',
|
|
1002
|
+
description: 'test1a'
|
|
1003
|
+
},
|
|
1004
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
1005
|
+
after: undefined
|
|
1014
1006
|
});
|
|
1007
|
+
const result3 = await writer.flush();
|
|
1015
1008
|
const checkpoint3 = result3.flushed_op;
|
|
1016
|
-
const
|
|
1009
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1010
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
|
|
1017
1011
|
const data = batch[0].chunkData.data.map((d) => {
|
|
1018
1012
|
return {
|
|
1019
1013
|
op: d.op,
|
|
@@ -1040,24 +1034,24 @@ bucket_definitions:
|
|
|
1040
1034
|
}
|
|
1041
1035
|
]);
|
|
1042
1036
|
}
|
|
1043
|
-
catch (
|
|
1044
|
-
|
|
1045
|
-
|
|
1037
|
+
catch (e_13) {
|
|
1038
|
+
env_13.error = e_13;
|
|
1039
|
+
env_13.hasError = true;
|
|
1046
1040
|
}
|
|
1047
1041
|
finally {
|
|
1048
|
-
const
|
|
1049
|
-
if (
|
|
1050
|
-
await
|
|
1042
|
+
const result_13 = __disposeResources(env_13);
|
|
1043
|
+
if (result_13)
|
|
1044
|
+
await result_13;
|
|
1051
1045
|
}
|
|
1052
1046
|
});
|
|
1053
1047
|
test('large batch', async () => {
|
|
1054
|
-
const
|
|
1048
|
+
const env_14 = { stack: [], error: void 0, hasError: false };
|
|
1055
1049
|
try {
|
|
1056
1050
|
// Test syncing a batch of data that is small in count,
|
|
1057
1051
|
// but large enough in size to be split over multiple returned batches.
|
|
1058
1052
|
// The specific batch splits is an implementation detail of the storage driver,
|
|
1059
1053
|
// and the test will have to updated when other implementations are added.
|
|
1060
|
-
const factory = __addDisposableResource(
|
|
1054
|
+
const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
|
|
1061
1055
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1062
1056
|
bucket_definitions:
|
|
1063
1057
|
global:
|
|
@@ -1067,54 +1061,54 @@ bucket_definitions:
|
|
|
1067
1061
|
storageVersion
|
|
1068
1062
|
}));
|
|
1069
1063
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1070
|
-
await bucketStorage.
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
});
|
|
1092
|
-
// Large enough to split the returned batch
|
|
1093
|
-
await batch.save({
|
|
1094
|
-
sourceTable,
|
|
1095
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1096
|
-
after: {
|
|
1097
|
-
id: 'large2',
|
|
1098
|
-
description: largeDescription
|
|
1099
|
-
},
|
|
1100
|
-
afterReplicaId: test_utils.rid('large2')
|
|
1101
|
-
});
|
|
1102
|
-
await batch.save({
|
|
1103
|
-
sourceTable,
|
|
1104
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1105
|
-
after: {
|
|
1106
|
-
id: 'test3',
|
|
1107
|
-
description: 'test3'
|
|
1108
|
-
},
|
|
1109
|
-
afterReplicaId: test_utils.rid('test3')
|
|
1110
|
-
});
|
|
1111
|
-
await batch.commit('1/1');
|
|
1064
|
+
const writer = __addDisposableResource(env_14, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1065
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1066
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1067
|
+
const largeDescription = '0123456789'.repeat(12_000_00);
|
|
1068
|
+
await writer.save({
|
|
1069
|
+
sourceTable,
|
|
1070
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1071
|
+
after: {
|
|
1072
|
+
id: 'test1',
|
|
1073
|
+
description: 'test1'
|
|
1074
|
+
},
|
|
1075
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1076
|
+
});
|
|
1077
|
+
await writer.save({
|
|
1078
|
+
sourceTable,
|
|
1079
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1080
|
+
after: {
|
|
1081
|
+
id: 'large1',
|
|
1082
|
+
description: largeDescription
|
|
1083
|
+
},
|
|
1084
|
+
afterReplicaId: test_utils.rid('large1')
|
|
1112
1085
|
});
|
|
1086
|
+
// Large enough to split the returned batch
|
|
1087
|
+
await writer.save({
|
|
1088
|
+
sourceTable,
|
|
1089
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1090
|
+
after: {
|
|
1091
|
+
id: 'large2',
|
|
1092
|
+
description: largeDescription
|
|
1093
|
+
},
|
|
1094
|
+
afterReplicaId: test_utils.rid('large2')
|
|
1095
|
+
});
|
|
1096
|
+
await writer.save({
|
|
1097
|
+
sourceTable,
|
|
1098
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1099
|
+
after: {
|
|
1100
|
+
id: 'test3',
|
|
1101
|
+
description: 'test3'
|
|
1102
|
+
},
|
|
1103
|
+
afterReplicaId: test_utils.rid('test3')
|
|
1104
|
+
});
|
|
1105
|
+
await writer.commit('1/1');
|
|
1113
1106
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1114
1107
|
const options = {
|
|
1115
1108
|
chunkLimitBytes: 16 * 1024 * 1024
|
|
1116
1109
|
};
|
|
1117
|
-
const
|
|
1110
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1111
|
+
const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
|
|
1118
1112
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
1119
1113
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
1120
1114
|
{ op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
|
|
@@ -1124,7 +1118,7 @@ bucket_definitions:
|
|
|
1124
1118
|
has_more: true,
|
|
1125
1119
|
next_after: '2'
|
|
1126
1120
|
});
|
|
1127
|
-
const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
1121
|
+
const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
|
|
1128
1122
|
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
1129
1123
|
{ op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
|
|
1130
1124
|
{ op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
|
|
@@ -1134,25 +1128,25 @@ bucket_definitions:
|
|
|
1134
1128
|
has_more: false,
|
|
1135
1129
|
next_after: '4'
|
|
1136
1130
|
});
|
|
1137
|
-
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
1131
|
+
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
|
|
1138
1132
|
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
1139
1133
|
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
1140
1134
|
}
|
|
1141
|
-
catch (
|
|
1142
|
-
|
|
1143
|
-
|
|
1135
|
+
catch (e_14) {
|
|
1136
|
+
env_14.error = e_14;
|
|
1137
|
+
env_14.hasError = true;
|
|
1144
1138
|
}
|
|
1145
1139
|
finally {
|
|
1146
|
-
const
|
|
1147
|
-
if (
|
|
1148
|
-
await
|
|
1140
|
+
const result_14 = __disposeResources(env_14);
|
|
1141
|
+
if (result_14)
|
|
1142
|
+
await result_14;
|
|
1149
1143
|
}
|
|
1150
1144
|
});
|
|
1151
1145
|
test('long batch', async () => {
|
|
1152
|
-
const
|
|
1146
|
+
const env_15 = { stack: [], error: void 0, hasError: false };
|
|
1153
1147
|
try {
|
|
1154
1148
|
// Test syncing a batch of data that is limited by count.
|
|
1155
|
-
const factory = __addDisposableResource(
|
|
1149
|
+
const factory = __addDisposableResource(env_15, await generateStorageFactory(), true);
|
|
1156
1150
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1157
1151
|
bucket_definitions:
|
|
1158
1152
|
global:
|
|
@@ -1162,24 +1156,24 @@ bucket_definitions:
|
|
|
1162
1156
|
storageVersion
|
|
1163
1157
|
}));
|
|
1164
1158
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1165
|
-
await bucketStorage.
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
});
|
|
1159
|
+
const writer = __addDisposableResource(env_15, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1160
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1161
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1162
|
+
for (let i = 1; i <= 6; i++) {
|
|
1163
|
+
await writer.save({
|
|
1164
|
+
sourceTable,
|
|
1165
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1166
|
+
after: {
|
|
1167
|
+
id: `test${i}`,
|
|
1168
|
+
description: `test${i}`
|
|
1169
|
+
},
|
|
1170
|
+
afterReplicaId: `test${i}`
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
await writer.commit('1/1');
|
|
1181
1174
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1182
|
-
const
|
|
1175
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1176
|
+
const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
|
|
1183
1177
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
1184
1178
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
1185
1179
|
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
@@ -1191,7 +1185,7 @@ bucket_definitions:
|
|
|
1191
1185
|
has_more: true,
|
|
1192
1186
|
next_after: '4'
|
|
1193
1187
|
});
|
|
1194
|
-
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
1188
|
+
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
1195
1189
|
limit: 4
|
|
1196
1190
|
}));
|
|
1197
1191
|
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
@@ -1203,27 +1197,27 @@ bucket_definitions:
|
|
|
1203
1197
|
has_more: false,
|
|
1204
1198
|
next_after: '6'
|
|
1205
1199
|
});
|
|
1206
|
-
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint,
|
|
1200
|
+
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
|
|
1207
1201
|
limit: 4
|
|
1208
1202
|
}));
|
|
1209
1203
|
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
1210
1204
|
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
1211
1205
|
}
|
|
1212
|
-
catch (
|
|
1213
|
-
|
|
1214
|
-
|
|
1206
|
+
catch (e_15) {
|
|
1207
|
+
env_15.error = e_15;
|
|
1208
|
+
env_15.hasError = true;
|
|
1215
1209
|
}
|
|
1216
1210
|
finally {
|
|
1217
|
-
const
|
|
1218
|
-
if (
|
|
1219
|
-
await
|
|
1211
|
+
const result_15 = __disposeResources(env_15);
|
|
1212
|
+
if (result_15)
|
|
1213
|
+
await result_15;
|
|
1220
1214
|
}
|
|
1221
1215
|
});
|
|
1222
1216
|
describe('batch has_more', () => {
|
|
1223
1217
|
const setup = async (options) => {
|
|
1224
|
-
const
|
|
1218
|
+
const env_16 = { stack: [], error: void 0, hasError: false };
|
|
1225
1219
|
try {
|
|
1226
|
-
const factory = __addDisposableResource(
|
|
1220
|
+
const factory = __addDisposableResource(env_16, await generateStorageFactory(), true);
|
|
1227
1221
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1228
1222
|
bucket_definitions:
|
|
1229
1223
|
global1:
|
|
@@ -1234,45 +1228,43 @@ bucket_definitions:
|
|
|
1234
1228
|
- SELECT id, description FROM test WHERE bucket = 'global2'
|
|
1235
1229
|
`, { storageVersion }));
|
|
1236
1230
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1237
|
-
await bucketStorage.
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
});
|
|
1231
|
+
const writer = __addDisposableResource(env_16, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1232
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1233
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1234
|
+
for (let i = 1; i <= 10; i++) {
|
|
1235
|
+
await writer.save({
|
|
1236
|
+
sourceTable,
|
|
1237
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1238
|
+
after: {
|
|
1239
|
+
id: `test${i}`,
|
|
1240
|
+
description: `test${i}`,
|
|
1241
|
+
bucket: i == 1 ? 'global1' : 'global2'
|
|
1242
|
+
},
|
|
1243
|
+
afterReplicaId: `test${i}`
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
await writer.commit('1/1');
|
|
1254
1247
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1255
|
-
const
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
return { syncRules, batch };
|
|
1248
|
+
const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
|
|
1249
|
+
const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
|
|
1250
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
|
|
1251
|
+
return { batch, global1Request, global2Request };
|
|
1260
1252
|
}
|
|
1261
|
-
catch (
|
|
1262
|
-
|
|
1263
|
-
|
|
1253
|
+
catch (e_16) {
|
|
1254
|
+
env_16.error = e_16;
|
|
1255
|
+
env_16.hasError = true;
|
|
1264
1256
|
}
|
|
1265
1257
|
finally {
|
|
1266
|
-
const
|
|
1267
|
-
if (
|
|
1268
|
-
await
|
|
1258
|
+
const result_16 = __disposeResources(env_16);
|
|
1259
|
+
if (result_16)
|
|
1260
|
+
await result_16;
|
|
1269
1261
|
}
|
|
1270
1262
|
};
|
|
1271
1263
|
test('batch has_more (1)', async () => {
|
|
1272
|
-
const { batch,
|
|
1264
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 5 });
|
|
1273
1265
|
expect(batch.length).toEqual(2);
|
|
1274
|
-
expect(batch[0].chunkData.bucket).toEqual(
|
|
1275
|
-
expect(batch[1].chunkData.bucket).toEqual(
|
|
1266
|
+
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1267
|
+
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1276
1268
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1277
1269
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1278
1270
|
]);
|
|
@@ -1294,10 +1286,10 @@ bucket_definitions:
|
|
|
1294
1286
|
});
|
|
1295
1287
|
});
|
|
1296
1288
|
test('batch has_more (2)', async () => {
|
|
1297
|
-
const { batch,
|
|
1289
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 11 });
|
|
1298
1290
|
expect(batch.length).toEqual(2);
|
|
1299
|
-
expect(batch[0].chunkData.bucket).toEqual(
|
|
1300
|
-
expect(batch[1].chunkData.bucket).toEqual(
|
|
1291
|
+
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1292
|
+
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1301
1293
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1302
1294
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1303
1295
|
]);
|
|
@@ -1325,11 +1317,11 @@ bucket_definitions:
|
|
|
1325
1317
|
});
|
|
1326
1318
|
test('batch has_more (3)', async () => {
|
|
1327
1319
|
// 50 bytes is more than 1 row, less than 2 rows
|
|
1328
|
-
const { batch,
|
|
1320
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
|
|
1329
1321
|
expect(batch.length).toEqual(3);
|
|
1330
|
-
expect(batch[0].chunkData.bucket).toEqual(
|
|
1331
|
-
expect(batch[1].chunkData.bucket).toEqual(
|
|
1332
|
-
expect(batch[2].chunkData.bucket).toEqual(
|
|
1322
|
+
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1323
|
+
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1324
|
+
expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1333
1325
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1334
1326
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
1335
1327
|
]);
|
|
@@ -1357,9 +1349,9 @@ bucket_definitions:
|
|
|
1357
1349
|
});
|
|
1358
1350
|
});
|
|
1359
1351
|
test('empty storage metrics', async () => {
|
|
1360
|
-
const
|
|
1352
|
+
const env_17 = { stack: [], error: void 0, hasError: false };
|
|
1361
1353
|
try {
|
|
1362
|
-
const f = __addDisposableResource(
|
|
1354
|
+
const f = __addDisposableResource(env_17, await generateStorageFactory({ dropAll: true }), true);
|
|
1363
1355
|
const metrics = await f.getStorageMetrics();
|
|
1364
1356
|
expect(metrics).toEqual({
|
|
1365
1357
|
operations_size_bytes: 0,
|
|
@@ -1368,29 +1360,28 @@ bucket_definitions:
|
|
|
1368
1360
|
});
|
|
1369
1361
|
const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
|
|
1370
1362
|
const storage = f.getInstance(r.persisted_sync_rules);
|
|
1371
|
-
await storage.
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
});
|
|
1363
|
+
const writer = __addDisposableResource(env_17, await storage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1364
|
+
await writer.markAllSnapshotDone('1/0');
|
|
1365
|
+
await writer.keepalive('1/0');
|
|
1375
1366
|
await f.getStorageMetrics();
|
|
1376
1367
|
}
|
|
1377
|
-
catch (
|
|
1378
|
-
|
|
1379
|
-
|
|
1368
|
+
catch (e_17) {
|
|
1369
|
+
env_17.error = e_17;
|
|
1370
|
+
env_17.hasError = true;
|
|
1380
1371
|
}
|
|
1381
1372
|
finally {
|
|
1382
|
-
const
|
|
1383
|
-
if (
|
|
1384
|
-
await
|
|
1373
|
+
const result_17 = __disposeResources(env_17);
|
|
1374
|
+
if (result_17)
|
|
1375
|
+
await result_17;
|
|
1385
1376
|
}
|
|
1386
1377
|
});
|
|
1387
1378
|
test('op_id initialization edge case', async () => {
|
|
1388
|
-
const
|
|
1379
|
+
const env_18 = { stack: [], error: void 0, hasError: false };
|
|
1389
1380
|
try {
|
|
1390
1381
|
// Test syncing a batch of data that is small in count,
|
|
1391
1382
|
// but large enough in size to be split over multiple returned chunks.
|
|
1392
1383
|
// Similar to the above test, but splits over 1MB chunks.
|
|
1393
|
-
const factory = __addDisposableResource(
|
|
1384
|
+
const factory = __addDisposableResource(env_18, await generateStorageFactory(), true);
|
|
1394
1385
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1395
1386
|
bucket_definitions:
|
|
1396
1387
|
global:
|
|
@@ -1401,50 +1392,49 @@ bucket_definitions:
|
|
|
1401
1392
|
storageVersion
|
|
1402
1393
|
}));
|
|
1403
1394
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1404
|
-
const
|
|
1405
|
-
const
|
|
1406
|
-
const
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
});
|
|
1395
|
+
const writer = __addDisposableResource(env_18, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1396
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
|
|
1397
|
+
const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
|
|
1398
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1399
|
+
// This saves a record to current_data, but not bucket_data.
|
|
1400
|
+
// This causes a checkpoint to be created without increasing the op_id sequence.
|
|
1401
|
+
await writer.save({
|
|
1402
|
+
sourceTable: sourceTableIgnore,
|
|
1403
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1404
|
+
after: {
|
|
1405
|
+
id: 'test1'
|
|
1406
|
+
},
|
|
1407
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1418
1408
|
});
|
|
1409
|
+
const result1 = await writer.flush();
|
|
1419
1410
|
const checkpoint1 = result1.flushed_op;
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
afterReplicaId: test_utils.rid('test2')
|
|
1428
|
-
});
|
|
1411
|
+
await writer.save({
|
|
1412
|
+
sourceTable: sourceTable,
|
|
1413
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1414
|
+
after: {
|
|
1415
|
+
id: 'test2'
|
|
1416
|
+
},
|
|
1417
|
+
afterReplicaId: test_utils.rid('test2')
|
|
1429
1418
|
});
|
|
1419
|
+
const result2 = await writer.flush();
|
|
1430
1420
|
const checkpoint2 = result2.flushed_op;
|
|
1431
1421
|
// we expect 0n and 1n, or 1n and 2n.
|
|
1432
1422
|
expect(checkpoint2).toBeGreaterThan(checkpoint1);
|
|
1433
1423
|
}
|
|
1434
|
-
catch (
|
|
1435
|
-
|
|
1436
|
-
|
|
1424
|
+
catch (e_18) {
|
|
1425
|
+
env_18.error = e_18;
|
|
1426
|
+
env_18.hasError = true;
|
|
1437
1427
|
}
|
|
1438
1428
|
finally {
|
|
1439
|
-
const
|
|
1440
|
-
if (
|
|
1441
|
-
await
|
|
1429
|
+
const result_18 = __disposeResources(env_18);
|
|
1430
|
+
if (result_18)
|
|
1431
|
+
await result_18;
|
|
1442
1432
|
}
|
|
1443
1433
|
});
|
|
1444
1434
|
test('unchanged checksums', async () => {
|
|
1445
|
-
const
|
|
1435
|
+
const env_19 = { stack: [], error: void 0, hasError: false };
|
|
1446
1436
|
try {
|
|
1447
|
-
const factory = __addDisposableResource(
|
|
1437
|
+
const factory = __addDisposableResource(env_19, await generateStorageFactory(), true);
|
|
1448
1438
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1449
1439
|
bucket_definitions:
|
|
1450
1440
|
global:
|
|
@@ -1454,49 +1444,41 @@ bucket_definitions:
|
|
|
1454
1444
|
storageVersion
|
|
1455
1445
|
}));
|
|
1456
1446
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1457
|
-
const
|
|
1458
|
-
await
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
});
|
|
1469
|
-
await batch.commit('1/1');
|
|
1447
|
+
const writer = __addDisposableResource(env_19, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1448
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1449
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1450
|
+
await writer.save({
|
|
1451
|
+
sourceTable,
|
|
1452
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1453
|
+
after: {
|
|
1454
|
+
id: 'test1',
|
|
1455
|
+
description: 'test1a'
|
|
1456
|
+
},
|
|
1457
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1470
1458
|
});
|
|
1459
|
+
await writer.commit('1/1');
|
|
1471
1460
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1472
|
-
const
|
|
1473
|
-
|
|
1474
|
-
];
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
]);
|
|
1478
|
-
const checksums2 = [
|
|
1479
|
-
...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
|
|
1480
|
-
];
|
|
1481
|
-
expect(checksums2).toEqual([
|
|
1482
|
-
{ bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
|
|
1483
|
-
]);
|
|
1461
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1462
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
1463
|
+
expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1464
|
+
const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
|
|
1465
|
+
expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1484
1466
|
}
|
|
1485
|
-
catch (
|
|
1486
|
-
|
|
1487
|
-
|
|
1467
|
+
catch (e_19) {
|
|
1468
|
+
env_19.error = e_19;
|
|
1469
|
+
env_19.hasError = true;
|
|
1488
1470
|
}
|
|
1489
1471
|
finally {
|
|
1490
|
-
const
|
|
1491
|
-
if (
|
|
1492
|
-
await
|
|
1472
|
+
const result_19 = __disposeResources(env_19);
|
|
1473
|
+
if (result_19)
|
|
1474
|
+
await result_19;
|
|
1493
1475
|
}
|
|
1494
1476
|
});
|
|
1495
1477
|
testChecksumBatching(config);
|
|
1496
1478
|
test('empty checkpoints (1)', async () => {
|
|
1497
|
-
const
|
|
1479
|
+
const env_20 = { stack: [], error: void 0, hasError: false };
|
|
1498
1480
|
try {
|
|
1499
|
-
const factory = __addDisposableResource(
|
|
1481
|
+
const factory = __addDisposableResource(env_20, await generateStorageFactory(), true);
|
|
1500
1482
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1501
1483
|
bucket_definitions:
|
|
1502
1484
|
global:
|
|
@@ -1504,38 +1486,37 @@ bucket_definitions:
|
|
|
1504
1486
|
- SELECT id, description FROM "%"
|
|
1505
1487
|
`, { storageVersion }));
|
|
1506
1488
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1507
|
-
await bucketStorage.
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
});
|
|
1489
|
+
const writer = __addDisposableResource(env_20, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1490
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1491
|
+
await writer.commit('1/1');
|
|
1492
|
+
const cp1 = await bucketStorage.getCheckpoint();
|
|
1493
|
+
expect(cp1.lsn).toEqual('1/1');
|
|
1494
|
+
await writer.commit('2/1', { createEmptyCheckpoints: true });
|
|
1495
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1496
|
+
expect(cp2.lsn).toEqual('2/1');
|
|
1497
|
+
await writer.keepalive('3/1');
|
|
1498
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1499
|
+
expect(cp3.lsn).toEqual('3/1');
|
|
1500
|
+
// For the last one, we skip creating empty checkpoints
|
|
1501
|
+
// This means the LSN stays at 3/1.
|
|
1502
|
+
await writer.commit('4/1', { createEmptyCheckpoints: false });
|
|
1503
|
+
const cp4 = await bucketStorage.getCheckpoint();
|
|
1504
|
+
expect(cp4.lsn).toEqual('3/1');
|
|
1524
1505
|
}
|
|
1525
|
-
catch (
|
|
1526
|
-
|
|
1527
|
-
|
|
1506
|
+
catch (e_20) {
|
|
1507
|
+
env_20.error = e_20;
|
|
1508
|
+
env_20.hasError = true;
|
|
1528
1509
|
}
|
|
1529
1510
|
finally {
|
|
1530
|
-
const
|
|
1531
|
-
if (
|
|
1532
|
-
await
|
|
1511
|
+
const result_20 = __disposeResources(env_20);
|
|
1512
|
+
if (result_20)
|
|
1513
|
+
await result_20;
|
|
1533
1514
|
}
|
|
1534
1515
|
});
|
|
1535
1516
|
test('empty checkpoints (2)', async () => {
|
|
1536
|
-
const
|
|
1517
|
+
const env_21 = { stack: [], error: void 0, hasError: false };
|
|
1537
1518
|
try {
|
|
1538
|
-
const factory = __addDisposableResource(
|
|
1519
|
+
const factory = __addDisposableResource(env_21, await generateStorageFactory(), true);
|
|
1539
1520
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1540
1521
|
bucket_definitions:
|
|
1541
1522
|
global:
|
|
@@ -1545,50 +1526,48 @@ bucket_definitions:
|
|
|
1545
1526
|
storageVersion
|
|
1546
1527
|
}));
|
|
1547
1528
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1548
|
-
const
|
|
1549
|
-
|
|
1550
|
-
await
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
});
|
|
1566
|
-
// This simulates what happens on a snapshot processor.
|
|
1567
|
-
// This may later change to a flush() rather than commit().
|
|
1568
|
-
await batch2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
|
|
1569
|
-
const cp3 = await bucketStorage.getCheckpoint();
|
|
1570
|
-
expect(cp3.lsn).toEqual('1/1'); // Still unchanged
|
|
1571
|
-
// This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
|
|
1572
|
-
await batch1.commit('4/1', { createEmptyCheckpoints: false });
|
|
1573
|
-
const cp4 = await bucketStorage.getCheckpoint();
|
|
1574
|
-
expect(cp4.lsn).toEqual('4/1');
|
|
1575
|
-
});
|
|
1529
|
+
const writer1 = __addDisposableResource(env_21, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1530
|
+
const writer2 = __addDisposableResource(env_21, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1531
|
+
const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
1532
|
+
// We simulate two concurrent batches, but sequential calls are enough for this test.
|
|
1533
|
+
await writer1.markAllSnapshotDone('1/1');
|
|
1534
|
+
await writer1.commit('1/1');
|
|
1535
|
+
await writer1.commit('2/1', { createEmptyCheckpoints: false });
|
|
1536
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1537
|
+
expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
|
|
1538
|
+
await writer2.save({
|
|
1539
|
+
sourceTable,
|
|
1540
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1541
|
+
after: {
|
|
1542
|
+
id: 'test1',
|
|
1543
|
+
description: 'test1a'
|
|
1544
|
+
},
|
|
1545
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1576
1546
|
});
|
|
1547
|
+
// This simulates what happens on a snapshot processor.
|
|
1548
|
+
// This may later change to a flush() rather than commit().
|
|
1549
|
+
await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
|
|
1550
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1551
|
+
expect(cp3.lsn).toEqual('1/1'); // Still unchanged
|
|
1552
|
+
// This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
|
|
1553
|
+
await writer1.commit('4/1', { createEmptyCheckpoints: false });
|
|
1554
|
+
const cp4 = await bucketStorage.getCheckpoint();
|
|
1555
|
+
expect(cp4.lsn).toEqual('4/1');
|
|
1577
1556
|
}
|
|
1578
|
-
catch (
|
|
1579
|
-
|
|
1580
|
-
|
|
1557
|
+
catch (e_21) {
|
|
1558
|
+
env_21.error = e_21;
|
|
1559
|
+
env_21.hasError = true;
|
|
1581
1560
|
}
|
|
1582
1561
|
finally {
|
|
1583
|
-
const
|
|
1584
|
-
if (
|
|
1585
|
-
await
|
|
1562
|
+
const result_21 = __disposeResources(env_21);
|
|
1563
|
+
if (result_21)
|
|
1564
|
+
await result_21;
|
|
1586
1565
|
}
|
|
1587
1566
|
});
|
|
1588
1567
|
test('empty checkpoints (sync rule activation)', async () => {
|
|
1589
|
-
const
|
|
1568
|
+
const env_22 = { stack: [], error: void 0, hasError: false };
|
|
1590
1569
|
try {
|
|
1591
|
-
const factory = __addDisposableResource(
|
|
1570
|
+
const factory = __addDisposableResource(env_22, await generateStorageFactory(), true);
|
|
1592
1571
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1593
1572
|
bucket_definitions:
|
|
1594
1573
|
global:
|
|
@@ -1598,54 +1577,47 @@ bucket_definitions:
|
|
|
1598
1577
|
storageVersion
|
|
1599
1578
|
}));
|
|
1600
1579
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1601
|
-
await bucketStorage.
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
await
|
|
1608
|
-
|
|
1609
|
-
const result = await batch.commit('2/1', { createEmptyCheckpoints: false });
|
|
1610
|
-
expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1611
|
-
});
|
|
1580
|
+
const writer = __addDisposableResource(env_22, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1581
|
+
const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
|
|
1582
|
+
expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1583
|
+
// Snapshot is only valid once we reach 3/1
|
|
1584
|
+
await writer.markAllSnapshotDone('3/1');
|
|
1585
|
+
// 2/1 < 3/1 - snapshot not valid yet, block checkpoint
|
|
1586
|
+
const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
|
|
1587
|
+
expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1612
1588
|
// No empty checkpoint should be created by the commit above.
|
|
1613
1589
|
const cp1 = await bucketStorage.getCheckpoint();
|
|
1614
1590
|
expect(cp1.lsn).toEqual(null);
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true });
|
|
1620
|
-
});
|
|
1591
|
+
// After this commit, the snapshot should be valid.
|
|
1592
|
+
// We specifically check that this is done even if createEmptyCheckpoints: false.
|
|
1593
|
+
const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
|
|
1594
|
+
expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
|
|
1621
1595
|
// Now, the checkpoint should advance the sync rules active.
|
|
1622
1596
|
const cp2 = await bucketStorage.getCheckpoint();
|
|
1623
1597
|
expect(cp2.lsn).toEqual('3/1');
|
|
1624
1598
|
const activeSyncRules = await factory.getActiveSyncRulesContent();
|
|
1625
1599
|
expect(activeSyncRules?.id).toEqual(syncRules.id);
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false });
|
|
1630
|
-
});
|
|
1600
|
+
// At this point, it should be a truely empty checkpoint
|
|
1601
|
+
const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
|
|
1602
|
+
expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
|
|
1631
1603
|
// Unchanged
|
|
1632
1604
|
const cp3 = await bucketStorage.getCheckpoint();
|
|
1633
1605
|
expect(cp3.lsn).toEqual('3/1');
|
|
1634
1606
|
}
|
|
1635
|
-
catch (
|
|
1636
|
-
|
|
1637
|
-
|
|
1607
|
+
catch (e_22) {
|
|
1608
|
+
env_22.error = e_22;
|
|
1609
|
+
env_22.hasError = true;
|
|
1638
1610
|
}
|
|
1639
1611
|
finally {
|
|
1640
|
-
const
|
|
1641
|
-
if (
|
|
1642
|
-
await
|
|
1612
|
+
const result_22 = __disposeResources(env_22);
|
|
1613
|
+
if (result_22)
|
|
1614
|
+
await result_22;
|
|
1643
1615
|
}
|
|
1644
1616
|
});
|
|
1645
1617
|
test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
|
|
1646
|
-
const
|
|
1618
|
+
const env_23 = { stack: [], error: void 0, hasError: false };
|
|
1647
1619
|
try {
|
|
1648
|
-
const factory = __addDisposableResource(
|
|
1620
|
+
const factory = __addDisposableResource(env_23, await generateStorageFactory(), true);
|
|
1649
1621
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1650
1622
|
bucket_definitions:
|
|
1651
1623
|
global:
|
|
@@ -1655,49 +1627,51 @@ bucket_definitions:
|
|
|
1655
1627
|
storageVersion
|
|
1656
1628
|
}));
|
|
1657
1629
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1658
|
-
const
|
|
1659
|
-
|
|
1630
|
+
const snapshotWriter = __addDisposableResource(env_23, await bucketStorage.createWriter({
|
|
1631
|
+
...test_utils.BATCH_OPTIONS,
|
|
1632
|
+
skipExistingRows: true
|
|
1633
|
+
}), true);
|
|
1634
|
+
const streamingWriter = __addDisposableResource(env_23, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1635
|
+
const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
|
|
1636
|
+
const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
|
|
1637
|
+
// We simulate two concurrent batches; separate writers are enough for this test.
|
|
1660
1638
|
// For this test, we assume that we start with a row "test1", which is picked up by a snapshot
|
|
1661
1639
|
// query, right before the delete is streamed. But the snapshot query is only persisted _after_
|
|
1662
1640
|
// the delete is streamed, and we need to ensure that the streamed delete takes precedence.
|
|
1663
|
-
await
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
},
|
|
1681
|
-
afterReplicaId: test_utils.rid('test1')
|
|
1682
|
-
});
|
|
1683
|
-
await snapshotBatch.markAllSnapshotDone('3/1');
|
|
1684
|
-
await snapshotBatch.commit('1/1');
|
|
1685
|
-
await streamingBatch.keepalive('3/1');
|
|
1686
|
-
});
|
|
1641
|
+
await streamingWriter.save({
|
|
1642
|
+
sourceTable: streamingTable,
|
|
1643
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
1644
|
+
before: {
|
|
1645
|
+
id: 'test1'
|
|
1646
|
+
},
|
|
1647
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
1648
|
+
});
|
|
1649
|
+
await streamingWriter.commit('2/1');
|
|
1650
|
+
await snapshotWriter.save({
|
|
1651
|
+
sourceTable: snapshotTable,
|
|
1652
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1653
|
+
after: {
|
|
1654
|
+
id: 'test1',
|
|
1655
|
+
description: 'test1a'
|
|
1656
|
+
},
|
|
1657
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1687
1658
|
});
|
|
1659
|
+
await snapshotWriter.markAllSnapshotDone('3/1');
|
|
1660
|
+
await snapshotWriter.commit('1/1');
|
|
1661
|
+
await streamingWriter.keepalive('3/1');
|
|
1688
1662
|
const cp = await bucketStorage.getCheckpoint();
|
|
1689
1663
|
expect(cp.lsn).toEqual('3/1');
|
|
1690
|
-
const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint,
|
|
1664
|
+
const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]));
|
|
1691
1665
|
expect(data).toEqual([]);
|
|
1692
1666
|
}
|
|
1693
|
-
catch (
|
|
1694
|
-
|
|
1695
|
-
|
|
1667
|
+
catch (e_23) {
|
|
1668
|
+
env_23.error = e_23;
|
|
1669
|
+
env_23.hasError = true;
|
|
1696
1670
|
}
|
|
1697
1671
|
finally {
|
|
1698
|
-
const
|
|
1699
|
-
if (
|
|
1700
|
-
await
|
|
1672
|
+
const result_23 = __disposeResources(env_23);
|
|
1673
|
+
if (result_23)
|
|
1674
|
+
await result_23;
|
|
1701
1675
|
}
|
|
1702
1676
|
});
|
|
1703
1677
|
}
|
|
@@ -1709,9 +1683,9 @@ bucket_definitions:
|
|
|
1709
1683
|
export function testChecksumBatching(config) {
|
|
1710
1684
|
const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
|
|
1711
1685
|
test('checksums for multiple buckets', async () => {
|
|
1712
|
-
const
|
|
1686
|
+
const env_24 = { stack: [], error: void 0, hasError: false };
|
|
1713
1687
|
try {
|
|
1714
|
-
const factory = __addDisposableResource(
|
|
1688
|
+
const factory = __addDisposableResource(env_24, await config.factory(), true);
|
|
1715
1689
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1716
1690
|
bucket_definitions:
|
|
1717
1691
|
user:
|
|
@@ -1722,46 +1696,48 @@ bucket_definitions:
|
|
|
1722
1696
|
storageVersion
|
|
1723
1697
|
}));
|
|
1724
1698
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1725
|
-
const
|
|
1726
|
-
await
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
}
|
|
1699
|
+
const writer = __addDisposableResource(env_24, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1700
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1701
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1702
|
+
for (let u of ['u1', 'u2', 'u3', 'u4']) {
|
|
1703
|
+
for (let t of ['t1', 't2', 't3', 't4']) {
|
|
1704
|
+
const id = `${t}_${u}`;
|
|
1705
|
+
await writer.save({
|
|
1706
|
+
sourceTable,
|
|
1707
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1708
|
+
after: {
|
|
1709
|
+
id,
|
|
1710
|
+
description: `${t} description`,
|
|
1711
|
+
user_id: u
|
|
1712
|
+
},
|
|
1713
|
+
afterReplicaId: test_utils.rid(id)
|
|
1714
|
+
});
|
|
1742
1715
|
}
|
|
1743
|
-
|
|
1744
|
-
|
|
1716
|
+
}
|
|
1717
|
+
await writer.commit('1/1');
|
|
1745
1718
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1746
1719
|
bucketStorage.clearChecksumCache();
|
|
1747
|
-
const
|
|
1748
|
-
const
|
|
1720
|
+
const users = ['u1', 'u2', 'u3', 'u4'];
|
|
1721
|
+
const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
|
|
1722
|
+
const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
|
|
1723
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
|
|
1749
1724
|
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1725
|
+
const expected = bucketRequests.map((request, index) => ({
|
|
1726
|
+
bucket: request.bucket,
|
|
1727
|
+
count: 4,
|
|
1728
|
+
checksum: expectedChecksums[index]
|
|
1729
|
+
}));
|
|
1730
|
+
expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1731
|
+
expect(checksums).toEqual(expected);
|
|
1756
1732
|
}
|
|
1757
|
-
catch (
|
|
1758
|
-
|
|
1759
|
-
|
|
1733
|
+
catch (e_24) {
|
|
1734
|
+
env_24.error = e_24;
|
|
1735
|
+
env_24.hasError = true;
|
|
1760
1736
|
}
|
|
1761
1737
|
finally {
|
|
1762
|
-
const
|
|
1763
|
-
if (
|
|
1764
|
-
await
|
|
1738
|
+
const result_24 = __disposeResources(env_24);
|
|
1739
|
+
if (result_24)
|
|
1740
|
+
await result_24;
|
|
1765
1741
|
}
|
|
1766
1742
|
});
|
|
1767
1743
|
}
|