@powersync/service-core-tests 0.15.0 → 0.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/test-utils/general-utils.d.ts +13 -1
- package/dist/test-utils/general-utils.js +30 -1
- package/dist/test-utils/general-utils.js.map +1 -1
- package/dist/test-utils/stream_utils.js +2 -2
- package/dist/test-utils/stream_utils.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +266 -257
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +36 -57
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +839 -863
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +228 -236
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +81 -89
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +468 -462
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/package.json +3 -3
- package/src/test-utils/general-utils.ts +41 -2
- package/src/test-utils/stream_utils.ts +2 -2
- package/src/tests/register-compacting-tests.ts +279 -270
- package/src/tests/register-data-storage-checkpoint-tests.ts +36 -57
- package/src/tests/register-data-storage-data-tests.ts +673 -770
- package/src/tests/register-data-storage-parameter-tests.ts +245 -257
- package/src/tests/register-parameter-compacting-tests.ts +84 -92
- package/src/tests/register-sync-tests.ts +375 -391
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
import { describe, expect, test } from 'vitest';
|
|
11
11
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
12
12
|
import { bucketRequest } from '../test-utils/test-utils-index.js';
|
|
13
|
-
import { bucketRequestMap, bucketRequests } from './util.js';
|
|
14
13
|
|
|
15
14
|
/**
|
|
16
15
|
* Normalize data from OplogEntries for comparison in tests.
|
|
@@ -37,8 +36,6 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig)
|
|
|
37
36
|
const generateStorageFactory = config.factory;
|
|
38
37
|
const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
|
|
39
38
|
|
|
40
|
-
const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config);
|
|
41
|
-
|
|
42
39
|
test('removing row', async () => {
|
|
43
40
|
await using factory = await generateStorageFactory();
|
|
44
41
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -53,33 +50,31 @@ bucket_definitions:
|
|
|
53
50
|
)
|
|
54
51
|
);
|
|
55
52
|
const bucketStorage = factory.getInstance(syncRules);
|
|
53
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
54
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
56
55
|
|
|
57
|
-
await
|
|
58
|
-
const sourceTable = TEST_TABLE;
|
|
59
|
-
await batch.markAllSnapshotDone('1/1');
|
|
56
|
+
await writer.markAllSnapshotDone('1/1');
|
|
60
57
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
});
|
|
70
|
-
await batch.save({
|
|
71
|
-
sourceTable,
|
|
72
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
73
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
74
|
-
});
|
|
75
|
-
await batch.commit('1/1');
|
|
58
|
+
await writer.save({
|
|
59
|
+
sourceTable: testTable,
|
|
60
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
61
|
+
after: {
|
|
62
|
+
id: 'test1',
|
|
63
|
+
description: 'test1'
|
|
64
|
+
},
|
|
65
|
+
afterReplicaId: test_utils.rid('test1')
|
|
76
66
|
});
|
|
67
|
+
await writer.save({
|
|
68
|
+
sourceTable: testTable,
|
|
69
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
70
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
71
|
+
});
|
|
72
|
+
await writer.commit('1/1');
|
|
77
73
|
|
|
78
74
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
79
75
|
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
);
|
|
76
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
77
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
83
78
|
const data = batch[0].chunkData.data.map((d) => {
|
|
84
79
|
return {
|
|
85
80
|
op: d.op,
|
|
@@ -96,12 +91,10 @@ bucket_definitions:
|
|
|
96
91
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
97
92
|
]);
|
|
98
93
|
|
|
99
|
-
const checksums = [
|
|
100
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
101
|
-
];
|
|
94
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
102
95
|
expect(checksums).toEqual([
|
|
103
96
|
{
|
|
104
|
-
bucket:
|
|
97
|
+
bucket: request.bucket,
|
|
105
98
|
checksum: (c1 + c2) & 0xffffffff,
|
|
106
99
|
count: 2
|
|
107
100
|
}
|
|
@@ -122,40 +115,33 @@ bucket_definitions:
|
|
|
122
115
|
)
|
|
123
116
|
);
|
|
124
117
|
const bucketStorage = factory.getInstance(syncRules);
|
|
125
|
-
|
|
126
|
-
const
|
|
127
|
-
await
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
await batch.commit('0/1');
|
|
118
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
119
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
120
|
+
await writer.markAllSnapshotDone('1/1');
|
|
121
|
+
|
|
122
|
+
await writer.save({
|
|
123
|
+
sourceTable: testTable,
|
|
124
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
125
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
137
126
|
});
|
|
138
127
|
|
|
139
|
-
await
|
|
140
|
-
const sourceTable = TEST_TABLE;
|
|
128
|
+
await writer.commit('0/1');
|
|
141
129
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
});
|
|
151
|
-
await batch.commit('2/1');
|
|
130
|
+
await writer.save({
|
|
131
|
+
sourceTable: testTable,
|
|
132
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
133
|
+
after: {
|
|
134
|
+
id: 'test1',
|
|
135
|
+
description: 'test1'
|
|
136
|
+
},
|
|
137
|
+
afterReplicaId: test_utils.rid('test1')
|
|
152
138
|
});
|
|
139
|
+
await writer.commit('2/1');
|
|
153
140
|
|
|
154
141
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
155
142
|
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
);
|
|
143
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
144
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
159
145
|
const data = batch[0].chunkData.data.map((d) => {
|
|
160
146
|
return {
|
|
161
147
|
op: d.op,
|
|
@@ -168,12 +154,10 @@ bucket_definitions:
|
|
|
168
154
|
|
|
169
155
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
170
156
|
|
|
171
|
-
const checksums = [
|
|
172
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
173
|
-
];
|
|
157
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
174
158
|
expect(checksums).toEqual([
|
|
175
159
|
{
|
|
176
|
-
bucket:
|
|
160
|
+
bucket: request.bucket,
|
|
177
161
|
checksum: c1 & 0xffffffff,
|
|
178
162
|
count: 1
|
|
179
163
|
}
|
|
@@ -195,44 +179,37 @@ bucket_definitions:
|
|
|
195
179
|
)
|
|
196
180
|
);
|
|
197
181
|
const bucketStorage = factory.getInstance(syncRules);
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
await
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
207
|
-
});
|
|
208
|
-
|
|
209
|
-
await batch.commit('0/1');
|
|
182
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
183
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
184
|
+
await writer.markAllSnapshotDone('1/1');
|
|
185
|
+
|
|
186
|
+
await writer.save({
|
|
187
|
+
sourceTable: testTable,
|
|
188
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
189
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
210
190
|
});
|
|
211
191
|
|
|
212
|
-
await
|
|
213
|
-
const sourceTable = TEST_TABLE;
|
|
192
|
+
await writer.commit('0/1');
|
|
214
193
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
});
|
|
228
|
-
await batch.commit('2/1');
|
|
194
|
+
await writer.save({
|
|
195
|
+
sourceTable: testTable,
|
|
196
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
197
|
+
before: {
|
|
198
|
+
id: 'test1'
|
|
199
|
+
},
|
|
200
|
+
after: {
|
|
201
|
+
id: 'test1',
|
|
202
|
+
description: 'test1'
|
|
203
|
+
},
|
|
204
|
+
beforeReplicaId: test_utils.rid('test1'),
|
|
205
|
+
afterReplicaId: test_utils.rid('test1')
|
|
229
206
|
});
|
|
207
|
+
await writer.commit('2/1');
|
|
230
208
|
|
|
231
209
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
232
210
|
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
);
|
|
211
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
212
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
236
213
|
const data = batch[0].chunkData.data.map((d) => {
|
|
237
214
|
return {
|
|
238
215
|
op: d.op,
|
|
@@ -245,12 +222,10 @@ bucket_definitions:
|
|
|
245
222
|
|
|
246
223
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
247
224
|
|
|
248
|
-
const checksums = [
|
|
249
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
250
|
-
];
|
|
225
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
251
226
|
expect(checksums).toEqual([
|
|
252
227
|
{
|
|
253
|
-
bucket:
|
|
228
|
+
bucket: request.bucket,
|
|
254
229
|
checksum: c1 & 0xffffffff,
|
|
255
230
|
count: 1
|
|
256
231
|
}
|
|
@@ -273,33 +248,30 @@ bucket_definitions:
|
|
|
273
248
|
)
|
|
274
249
|
);
|
|
275
250
|
const bucketStorage = factory.getInstance(syncRules);
|
|
276
|
-
|
|
277
|
-
await
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
afterReplicaId: test_utils.rid('test1')
|
|
294
|
-
});
|
|
295
|
-
await batch.commit('1/1');
|
|
251
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
252
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
253
|
+
await writer.markAllSnapshotDone('1/1');
|
|
254
|
+
|
|
255
|
+
await writer.save({
|
|
256
|
+
sourceTable: testTable,
|
|
257
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
258
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
259
|
+
});
|
|
260
|
+
await writer.save({
|
|
261
|
+
sourceTable: testTable,
|
|
262
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
263
|
+
after: {
|
|
264
|
+
id: 'test1',
|
|
265
|
+
description: 'test1'
|
|
266
|
+
},
|
|
267
|
+
afterReplicaId: test_utils.rid('test1')
|
|
296
268
|
});
|
|
269
|
+
await writer.commit('1/1');
|
|
297
270
|
|
|
298
271
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
299
272
|
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
);
|
|
273
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
274
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
303
275
|
const data = batch[0].chunkData.data.map((d) => {
|
|
304
276
|
return {
|
|
305
277
|
op: d.op,
|
|
@@ -312,12 +284,10 @@ bucket_definitions:
|
|
|
312
284
|
|
|
313
285
|
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
314
286
|
|
|
315
|
-
const checksums = [
|
|
316
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
317
|
-
];
|
|
287
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
318
288
|
expect(checksums).toEqual([
|
|
319
289
|
{
|
|
320
|
-
bucket:
|
|
290
|
+
bucket: request.bucket,
|
|
321
291
|
checksum: c1 & 0xffffffff,
|
|
322
292
|
count: 1
|
|
323
293
|
}
|
|
@@ -340,12 +310,12 @@ bucket_definitions:
|
|
|
340
310
|
)
|
|
341
311
|
);
|
|
342
312
|
const bucketStorage = factory.getInstance(syncRules);
|
|
313
|
+
{
|
|
314
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
315
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
316
|
+
await writer.markAllSnapshotDone('1/1');
|
|
343
317
|
|
|
344
|
-
|
|
345
|
-
const sourceTable = TEST_TABLE;
|
|
346
|
-
await batch.markAllSnapshotDone('1/1');
|
|
347
|
-
|
|
348
|
-
await batch.save({
|
|
318
|
+
await writer.save({
|
|
349
319
|
sourceTable,
|
|
350
320
|
tag: storage.SaveOperationTag.INSERT,
|
|
351
321
|
after: {
|
|
@@ -354,12 +324,12 @@ bucket_definitions:
|
|
|
354
324
|
},
|
|
355
325
|
afterReplicaId: test_utils.rid('test1')
|
|
356
326
|
});
|
|
357
|
-
await
|
|
327
|
+
await writer.save({
|
|
358
328
|
sourceTable,
|
|
359
329
|
tag: storage.SaveOperationTag.DELETE,
|
|
360
330
|
beforeReplicaId: test_utils.rid('test1')
|
|
361
331
|
});
|
|
362
|
-
await
|
|
332
|
+
await writer.save({
|
|
363
333
|
sourceTable,
|
|
364
334
|
tag: storage.SaveOperationTag.INSERT,
|
|
365
335
|
after: {
|
|
@@ -368,26 +338,26 @@ bucket_definitions:
|
|
|
368
338
|
},
|
|
369
339
|
afterReplicaId: test_utils.rid('test1')
|
|
370
340
|
});
|
|
371
|
-
await
|
|
372
|
-
}
|
|
341
|
+
await writer.commit('1/1');
|
|
342
|
+
}
|
|
373
343
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
await
|
|
344
|
+
{
|
|
345
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
346
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
347
|
+
await writer.markAllSnapshotDone('1/1');
|
|
377
348
|
|
|
378
|
-
await
|
|
349
|
+
await writer.save({
|
|
379
350
|
sourceTable,
|
|
380
351
|
tag: storage.SaveOperationTag.DELETE,
|
|
381
352
|
beforeReplicaId: test_utils.rid('test1')
|
|
382
353
|
});
|
|
383
|
-
await
|
|
384
|
-
}
|
|
354
|
+
await writer.commit('2/1');
|
|
355
|
+
}
|
|
385
356
|
|
|
386
357
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
387
358
|
|
|
388
|
-
const
|
|
389
|
-
|
|
390
|
-
);
|
|
359
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
360
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
391
361
|
|
|
392
362
|
expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
|
|
393
363
|
|
|
@@ -418,47 +388,45 @@ bucket_definitions:
|
|
|
418
388
|
)
|
|
419
389
|
);
|
|
420
390
|
const bucketStorage = factory.getInstance(syncRules);
|
|
391
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
392
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
393
|
+
await writer.markAllSnapshotDone('1/1');
|
|
394
|
+
await writer.save({
|
|
395
|
+
sourceTable,
|
|
396
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
397
|
+
after: {
|
|
398
|
+
id: 'test1',
|
|
399
|
+
client_id: 'client1a',
|
|
400
|
+
description: 'test1a'
|
|
401
|
+
},
|
|
402
|
+
afterReplicaId: test_utils.rid('test1')
|
|
403
|
+
});
|
|
404
|
+
await writer.save({
|
|
405
|
+
sourceTable,
|
|
406
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
407
|
+
after: {
|
|
408
|
+
id: 'test1',
|
|
409
|
+
client_id: 'client1b',
|
|
410
|
+
description: 'test1b'
|
|
411
|
+
},
|
|
412
|
+
afterReplicaId: test_utils.rid('test1')
|
|
413
|
+
});
|
|
421
414
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
description: 'test1a'
|
|
432
|
-
},
|
|
433
|
-
afterReplicaId: test_utils.rid('test1')
|
|
434
|
-
});
|
|
435
|
-
await batch.save({
|
|
436
|
-
sourceTable,
|
|
437
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
438
|
-
after: {
|
|
439
|
-
id: 'test1',
|
|
440
|
-
client_id: 'client1b',
|
|
441
|
-
description: 'test1b'
|
|
442
|
-
},
|
|
443
|
-
afterReplicaId: test_utils.rid('test1')
|
|
444
|
-
});
|
|
445
|
-
|
|
446
|
-
await batch.save({
|
|
447
|
-
sourceTable,
|
|
448
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
449
|
-
after: {
|
|
450
|
-
id: 'test2',
|
|
451
|
-
client_id: 'client2',
|
|
452
|
-
description: 'test2'
|
|
453
|
-
},
|
|
454
|
-
afterReplicaId: test_utils.rid('test2')
|
|
455
|
-
});
|
|
456
|
-
|
|
457
|
-
await batch.commit('1/1');
|
|
415
|
+
await writer.save({
|
|
416
|
+
sourceTable,
|
|
417
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
418
|
+
after: {
|
|
419
|
+
id: 'test2',
|
|
420
|
+
client_id: 'client2',
|
|
421
|
+
description: 'test2'
|
|
422
|
+
},
|
|
423
|
+
afterReplicaId: test_utils.rid('test2')
|
|
458
424
|
});
|
|
425
|
+
|
|
426
|
+
await writer.commit('1/1');
|
|
459
427
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
460
428
|
const batch = await test_utils.fromAsync(
|
|
461
|
-
bucketStorage.getBucketDataBatch(checkpoint,
|
|
429
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')])
|
|
462
430
|
);
|
|
463
431
|
const data = batch[0].chunkData.data.map((d) => {
|
|
464
432
|
return {
|
|
@@ -491,49 +459,40 @@ bucket_definitions:
|
|
|
491
459
|
)
|
|
492
460
|
);
|
|
493
461
|
const bucketStorage = factory.getInstance(syncRules);
|
|
494
|
-
|
|
495
|
-
await
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
afterReplicaId: test_utils.rid('test1')
|
|
507
|
-
});
|
|
462
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
463
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
464
|
+
await writer.markAllSnapshotDone('1/1');
|
|
465
|
+
|
|
466
|
+
await writer.save({
|
|
467
|
+
sourceTable,
|
|
468
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
469
|
+
after: {
|
|
470
|
+
id: 'test1',
|
|
471
|
+
description: 'test1'
|
|
472
|
+
},
|
|
473
|
+
afterReplicaId: test_utils.rid('test1')
|
|
508
474
|
});
|
|
475
|
+
await writer.flush();
|
|
509
476
|
|
|
510
|
-
await
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
sourceTable,
|
|
515
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
516
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
517
|
-
});
|
|
518
|
-
|
|
519
|
-
await batch.commit('1/1');
|
|
477
|
+
await writer.save({
|
|
478
|
+
sourceTable,
|
|
479
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
480
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
520
481
|
});
|
|
521
482
|
|
|
522
|
-
await
|
|
523
|
-
const sourceTable = TEST_TABLE;
|
|
483
|
+
await writer.commit('1/1');
|
|
524
484
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
});
|
|
485
|
+
await writer.save({
|
|
486
|
+
sourceTable,
|
|
487
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
488
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
530
489
|
});
|
|
490
|
+
await writer.flush();
|
|
531
491
|
|
|
532
492
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
533
493
|
|
|
534
|
-
const
|
|
535
|
-
|
|
536
|
-
);
|
|
494
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
495
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
537
496
|
const data = batch[0].chunkData.data.map((d) => {
|
|
538
497
|
return {
|
|
539
498
|
op: d.op,
|
|
@@ -550,9 +509,7 @@ bucket_definitions:
|
|
|
550
509
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
551
510
|
]);
|
|
552
511
|
|
|
553
|
-
const checksums = [
|
|
554
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
555
|
-
];
|
|
512
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
556
513
|
expect(checksums).toEqual([
|
|
557
514
|
{
|
|
558
515
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -576,93 +533,85 @@ bucket_definitions:
|
|
|
576
533
|
)
|
|
577
534
|
);
|
|
578
535
|
const bucketStorage = factory.getInstance(syncRules);
|
|
579
|
-
|
|
580
|
-
await
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
afterReplicaId: test_utils.rid('test1')
|
|
592
|
-
});
|
|
536
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
537
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
538
|
+
await writer.markAllSnapshotDone('1/1');
|
|
539
|
+
|
|
540
|
+
await writer.save({
|
|
541
|
+
sourceTable,
|
|
542
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
543
|
+
after: {
|
|
544
|
+
id: 'test1',
|
|
545
|
+
description: 'test1'
|
|
546
|
+
},
|
|
547
|
+
afterReplicaId: test_utils.rid('test1')
|
|
593
548
|
});
|
|
549
|
+
await writer.flush();
|
|
594
550
|
|
|
595
|
-
await
|
|
596
|
-
await batch.markAllSnapshotDone('1/1');
|
|
597
|
-
const sourceTable = TEST_TABLE;
|
|
598
|
-
|
|
599
|
-
await batch.save({
|
|
600
|
-
sourceTable,
|
|
601
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
602
|
-
after: {
|
|
603
|
-
id: 'test1',
|
|
604
|
-
description: undefined
|
|
605
|
-
},
|
|
606
|
-
afterReplicaId: test_utils.rid('test1')
|
|
607
|
-
});
|
|
551
|
+
await writer.markAllSnapshotDone('1/1');
|
|
608
552
|
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
553
|
+
await writer.save({
|
|
554
|
+
sourceTable,
|
|
555
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
556
|
+
after: {
|
|
557
|
+
id: 'test1',
|
|
558
|
+
description: undefined
|
|
559
|
+
},
|
|
560
|
+
afterReplicaId: test_utils.rid('test1')
|
|
561
|
+
});
|
|
618
562
|
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
563
|
+
await writer.save({
|
|
564
|
+
sourceTable,
|
|
565
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
566
|
+
after: {
|
|
567
|
+
id: 'test1',
|
|
568
|
+
description: undefined
|
|
569
|
+
},
|
|
570
|
+
afterReplicaId: test_utils.rid('test1')
|
|
571
|
+
});
|
|
624
572
|
|
|
625
|
-
|
|
573
|
+
await writer.save({
|
|
574
|
+
sourceTable,
|
|
575
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
576
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
626
577
|
});
|
|
627
578
|
|
|
628
|
-
await
|
|
629
|
-
await batch.markAllSnapshotDone('1/1');
|
|
630
|
-
const sourceTable = TEST_TABLE;
|
|
579
|
+
await writer.commit('1/1');
|
|
631
580
|
|
|
632
|
-
|
|
633
|
-
sourceTable,
|
|
634
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
635
|
-
after: {
|
|
636
|
-
id: 'test1',
|
|
637
|
-
description: undefined
|
|
638
|
-
},
|
|
639
|
-
afterReplicaId: test_utils.rid('test1')
|
|
640
|
-
});
|
|
581
|
+
await writer.markAllSnapshotDone('1/1');
|
|
641
582
|
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
583
|
+
await writer.save({
|
|
584
|
+
sourceTable,
|
|
585
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
586
|
+
after: {
|
|
587
|
+
id: 'test1',
|
|
588
|
+
description: undefined
|
|
589
|
+
},
|
|
590
|
+
afterReplicaId: test_utils.rid('test1')
|
|
591
|
+
});
|
|
651
592
|
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
593
|
+
await writer.save({
|
|
594
|
+
sourceTable,
|
|
595
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
596
|
+
after: {
|
|
597
|
+
id: 'test1',
|
|
598
|
+
description: undefined
|
|
599
|
+
},
|
|
600
|
+
afterReplicaId: test_utils.rid('test1')
|
|
601
|
+
});
|
|
657
602
|
|
|
658
|
-
|
|
603
|
+
await writer.save({
|
|
604
|
+
sourceTable,
|
|
605
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
606
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
659
607
|
});
|
|
660
608
|
|
|
609
|
+
await writer.commit('2/1');
|
|
610
|
+
|
|
661
611
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
662
612
|
|
|
663
|
-
const
|
|
664
|
-
|
|
665
|
-
);
|
|
613
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
614
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
666
615
|
|
|
667
616
|
const data = batch[0].chunkData.data.map((d) => {
|
|
668
617
|
return {
|
|
@@ -682,9 +631,7 @@ bucket_definitions:
|
|
|
682
631
|
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
683
632
|
]);
|
|
684
633
|
|
|
685
|
-
const checksums = [
|
|
686
|
-
...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values()
|
|
687
|
-
];
|
|
634
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
688
635
|
expect(checksums).toEqual([
|
|
689
636
|
{
|
|
690
637
|
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
@@ -716,120 +663,117 @@ bucket_definitions:
|
|
|
716
663
|
)
|
|
717
664
|
);
|
|
718
665
|
const bucketStorage = factory.getInstance(syncRules);
|
|
666
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
667
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
719
668
|
|
|
720
669
|
// Pre-setup
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
afterReplicaId: test_utils.rid('test1')
|
|
733
|
-
});
|
|
670
|
+
await writer.markAllSnapshotDone('1/1');
|
|
671
|
+
|
|
672
|
+
await writer.save({
|
|
673
|
+
sourceTable,
|
|
674
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
675
|
+
after: {
|
|
676
|
+
id: 'test1',
|
|
677
|
+
description: 'test1a'
|
|
678
|
+
},
|
|
679
|
+
afterReplicaId: test_utils.rid('test1')
|
|
680
|
+
});
|
|
734
681
|
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
});
|
|
682
|
+
await writer.save({
|
|
683
|
+
sourceTable,
|
|
684
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
685
|
+
after: {
|
|
686
|
+
id: 'test2',
|
|
687
|
+
description: 'test2a'
|
|
688
|
+
},
|
|
689
|
+
afterReplicaId: test_utils.rid('test2')
|
|
744
690
|
});
|
|
691
|
+
const result1 = await writer.flush();
|
|
745
692
|
|
|
746
693
|
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
747
694
|
|
|
748
695
|
// Test batch
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
afterReplicaId: test_utils.rid('test1')
|
|
760
|
-
});
|
|
696
|
+
// b
|
|
697
|
+
await writer.save({
|
|
698
|
+
sourceTable,
|
|
699
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
700
|
+
after: {
|
|
701
|
+
id: 'test1',
|
|
702
|
+
description: 'test1b'
|
|
703
|
+
},
|
|
704
|
+
afterReplicaId: test_utils.rid('test1')
|
|
705
|
+
});
|
|
761
706
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
707
|
+
await writer.save({
|
|
708
|
+
sourceTable,
|
|
709
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
710
|
+
before: {
|
|
711
|
+
id: 'test1'
|
|
712
|
+
},
|
|
713
|
+
beforeReplicaId: test_utils.rid('test1'),
|
|
714
|
+
after: {
|
|
715
|
+
id: 'test2',
|
|
716
|
+
description: 'test2b'
|
|
717
|
+
},
|
|
718
|
+
afterReplicaId: test_utils.rid('test2')
|
|
719
|
+
});
|
|
775
720
|
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
721
|
+
await writer.save({
|
|
722
|
+
sourceTable,
|
|
723
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
724
|
+
before: {
|
|
725
|
+
id: 'test2'
|
|
726
|
+
},
|
|
727
|
+
beforeReplicaId: test_utils.rid('test2'),
|
|
728
|
+
after: {
|
|
729
|
+
id: 'test3',
|
|
730
|
+
description: 'test3b'
|
|
731
|
+
},
|
|
787
732
|
|
|
788
|
-
|
|
789
|
-
|
|
733
|
+
afterReplicaId: test_utils.rid('test3')
|
|
734
|
+
});
|
|
790
735
|
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
736
|
+
// c
|
|
737
|
+
await writer.save({
|
|
738
|
+
sourceTable,
|
|
739
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
740
|
+
after: {
|
|
741
|
+
id: 'test2',
|
|
742
|
+
description: 'test2c'
|
|
743
|
+
},
|
|
744
|
+
afterReplicaId: test_utils.rid('test2')
|
|
745
|
+
});
|
|
801
746
|
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
747
|
+
// d
|
|
748
|
+
await writer.save({
|
|
749
|
+
sourceTable,
|
|
750
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
751
|
+
after: {
|
|
752
|
+
id: 'test4',
|
|
753
|
+
description: 'test4d'
|
|
754
|
+
},
|
|
755
|
+
afterReplicaId: test_utils.rid('test4')
|
|
756
|
+
});
|
|
812
757
|
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
});
|
|
758
|
+
await writer.save({
|
|
759
|
+
sourceTable,
|
|
760
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
761
|
+
before: {
|
|
762
|
+
id: 'test4'
|
|
763
|
+
},
|
|
764
|
+
beforeReplicaId: test_utils.rid('test4'),
|
|
765
|
+
after: {
|
|
766
|
+
id: 'test5',
|
|
767
|
+
description: 'test5d'
|
|
768
|
+
},
|
|
769
|
+
afterReplicaId: test_utils.rid('test5')
|
|
826
770
|
});
|
|
771
|
+
const result2 = await writer.flush();
|
|
827
772
|
|
|
828
773
|
const checkpoint2 = result2!.flushed_op;
|
|
829
774
|
|
|
830
|
-
const
|
|
831
|
-
|
|
832
|
-
);
|
|
775
|
+
const request = bucketRequest(syncRules, 'global[]', checkpoint1);
|
|
776
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
|
|
833
777
|
|
|
834
778
|
const data = batch[0].chunkData.data.map((d) => {
|
|
835
779
|
return {
|
|
@@ -881,60 +825,59 @@ bucket_definitions:
|
|
|
881
825
|
);
|
|
882
826
|
const bucketStorage = factory.getInstance(syncRules);
|
|
883
827
|
|
|
828
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
884
829
|
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
885
830
|
|
|
886
831
|
// Pre-setup
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
afterReplicaId: rid2('test1', 'test1a')
|
|
897
|
-
});
|
|
832
|
+
await writer.markAllSnapshotDone('1/1');
|
|
833
|
+
await writer.save({
|
|
834
|
+
sourceTable,
|
|
835
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
836
|
+
after: {
|
|
837
|
+
id: 'test1',
|
|
838
|
+
description: 'test1a'
|
|
839
|
+
},
|
|
840
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
898
841
|
});
|
|
842
|
+
const result1 = await writer.flush();
|
|
899
843
|
|
|
900
844
|
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
901
845
|
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
afterReplicaId: rid2('test1', 'test1b')
|
|
917
|
-
});
|
|
846
|
+
// Unchanged, but has a before id
|
|
847
|
+
await writer.save({
|
|
848
|
+
sourceTable,
|
|
849
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
850
|
+
before: {
|
|
851
|
+
id: 'test1',
|
|
852
|
+
description: 'test1a'
|
|
853
|
+
},
|
|
854
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
855
|
+
after: {
|
|
856
|
+
id: 'test1',
|
|
857
|
+
description: 'test1b'
|
|
858
|
+
},
|
|
859
|
+
afterReplicaId: rid2('test1', 'test1b')
|
|
918
860
|
});
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
});
|
|
861
|
+
const result2 = await writer.flush();
|
|
862
|
+
|
|
863
|
+
// Delete
|
|
864
|
+
await writer.save({
|
|
865
|
+
sourceTable,
|
|
866
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
867
|
+
before: {
|
|
868
|
+
id: 'test1',
|
|
869
|
+
description: 'test1b'
|
|
870
|
+
},
|
|
871
|
+
beforeReplicaId: rid2('test1', 'test1b'),
|
|
872
|
+
after: undefined
|
|
932
873
|
});
|
|
874
|
+
const result3 = await writer.flush();
|
|
933
875
|
|
|
934
876
|
const checkpoint3 = result3!.flushed_op;
|
|
935
877
|
|
|
878
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
936
879
|
const batch = await test_utils.fromAsync(
|
|
937
|
-
bucketStorage.getBucketDataBatch(checkpoint3,
|
|
880
|
+
bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }])
|
|
938
881
|
);
|
|
939
882
|
const data = batch[0].chunkData.data.map((d) => {
|
|
940
883
|
return {
|
|
@@ -995,60 +938,59 @@ bucket_definitions:
|
|
|
995
938
|
);
|
|
996
939
|
const bucketStorage = factory.getInstance(syncRules);
|
|
997
940
|
|
|
941
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
998
942
|
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
999
943
|
|
|
1000
944
|
// Pre-setup
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
afterReplicaId: rid2('test1', 'test1a')
|
|
1011
|
-
});
|
|
945
|
+
await writer.markAllSnapshotDone('1/1');
|
|
946
|
+
await writer.save({
|
|
947
|
+
sourceTable,
|
|
948
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
949
|
+
after: {
|
|
950
|
+
id: 'test1',
|
|
951
|
+
description: 'test1a'
|
|
952
|
+
},
|
|
953
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
1012
954
|
});
|
|
955
|
+
const result1 = await writer.flush();
|
|
1013
956
|
|
|
1014
957
|
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
1015
958
|
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
afterReplicaId: rid2('test1', 'test1a')
|
|
1031
|
-
});
|
|
959
|
+
// Unchanged, but has a before id
|
|
960
|
+
await writer.save({
|
|
961
|
+
sourceTable,
|
|
962
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
963
|
+
before: {
|
|
964
|
+
id: 'test1',
|
|
965
|
+
description: 'test1a'
|
|
966
|
+
},
|
|
967
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
968
|
+
after: {
|
|
969
|
+
id: 'test1',
|
|
970
|
+
description: 'test1a'
|
|
971
|
+
},
|
|
972
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
1032
973
|
});
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
});
|
|
974
|
+
const result2 = await writer.flush();
|
|
975
|
+
|
|
976
|
+
// Delete
|
|
977
|
+
await writer.save({
|
|
978
|
+
sourceTable,
|
|
979
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
980
|
+
before: {
|
|
981
|
+
id: 'test1',
|
|
982
|
+
description: 'test1a'
|
|
983
|
+
},
|
|
984
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
985
|
+
after: undefined
|
|
1046
986
|
});
|
|
987
|
+
const result3 = await writer.flush();
|
|
1047
988
|
|
|
1048
989
|
const checkpoint3 = result3!.flushed_op;
|
|
1049
990
|
|
|
991
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1050
992
|
const batch = await test_utils.fromAsync(
|
|
1051
|
-
bucketStorage.getBucketDataBatch(checkpoint3,
|
|
993
|
+
bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }])
|
|
1052
994
|
);
|
|
1053
995
|
const data = batch[0].chunkData.data.map((d) => {
|
|
1054
996
|
return {
|
|
@@ -1098,66 +1040,63 @@ bucket_definitions:
|
|
|
1098
1040
|
)
|
|
1099
1041
|
);
|
|
1100
1042
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1043
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1044
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1045
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1046
|
+
|
|
1047
|
+
const largeDescription = '0123456789'.repeat(12_000_00);
|
|
1048
|
+
|
|
1049
|
+
await writer.save({
|
|
1050
|
+
sourceTable,
|
|
1051
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1052
|
+
after: {
|
|
1053
|
+
id: 'test1',
|
|
1054
|
+
description: 'test1'
|
|
1055
|
+
},
|
|
1056
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1057
|
+
});
|
|
1101
1058
|
|
|
1102
|
-
await
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
after: {
|
|
1112
|
-
id: 'test1',
|
|
1113
|
-
description: 'test1'
|
|
1114
|
-
},
|
|
1115
|
-
afterReplicaId: test_utils.rid('test1')
|
|
1116
|
-
});
|
|
1117
|
-
|
|
1118
|
-
await batch.save({
|
|
1119
|
-
sourceTable,
|
|
1120
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1121
|
-
after: {
|
|
1122
|
-
id: 'large1',
|
|
1123
|
-
description: largeDescription
|
|
1124
|
-
},
|
|
1125
|
-
afterReplicaId: test_utils.rid('large1')
|
|
1126
|
-
});
|
|
1127
|
-
|
|
1128
|
-
// Large enough to split the returned batch
|
|
1129
|
-
await batch.save({
|
|
1130
|
-
sourceTable,
|
|
1131
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1132
|
-
after: {
|
|
1133
|
-
id: 'large2',
|
|
1134
|
-
description: largeDescription
|
|
1135
|
-
},
|
|
1136
|
-
afterReplicaId: test_utils.rid('large2')
|
|
1137
|
-
});
|
|
1059
|
+
await writer.save({
|
|
1060
|
+
sourceTable,
|
|
1061
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1062
|
+
after: {
|
|
1063
|
+
id: 'large1',
|
|
1064
|
+
description: largeDescription
|
|
1065
|
+
},
|
|
1066
|
+
afterReplicaId: test_utils.rid('large1')
|
|
1067
|
+
});
|
|
1138
1068
|
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1069
|
+
// Large enough to split the returned batch
|
|
1070
|
+
await writer.save({
|
|
1071
|
+
sourceTable,
|
|
1072
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1073
|
+
after: {
|
|
1074
|
+
id: 'large2',
|
|
1075
|
+
description: largeDescription
|
|
1076
|
+
},
|
|
1077
|
+
afterReplicaId: test_utils.rid('large2')
|
|
1078
|
+
});
|
|
1148
1079
|
|
|
1149
|
-
|
|
1080
|
+
await writer.save({
|
|
1081
|
+
sourceTable,
|
|
1082
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1083
|
+
after: {
|
|
1084
|
+
id: 'test3',
|
|
1085
|
+
description: 'test3'
|
|
1086
|
+
},
|
|
1087
|
+
afterReplicaId: test_utils.rid('test3')
|
|
1150
1088
|
});
|
|
1151
1089
|
|
|
1090
|
+
await writer.commit('1/1');
|
|
1091
|
+
|
|
1152
1092
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1153
1093
|
|
|
1154
1094
|
const options: storage.BucketDataBatchOptions = {
|
|
1155
1095
|
chunkLimitBytes: 16 * 1024 * 1024
|
|
1156
1096
|
};
|
|
1157
1097
|
|
|
1158
|
-
const
|
|
1159
|
-
|
|
1160
|
-
);
|
|
1098
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1099
|
+
const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
|
|
1161
1100
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
1162
1101
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
1163
1102
|
{ op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
|
|
@@ -1171,7 +1110,7 @@ bucket_definitions:
|
|
|
1171
1110
|
const batch2 = await test_utils.fromAsync(
|
|
1172
1111
|
bucketStorage.getBucketDataBatch(
|
|
1173
1112
|
checkpoint,
|
|
1174
|
-
|
|
1113
|
+
[{ ...request, start: BigInt(batch1[0].chunkData.next_after) }],
|
|
1175
1114
|
options
|
|
1176
1115
|
)
|
|
1177
1116
|
);
|
|
@@ -1188,7 +1127,7 @@ bucket_definitions:
|
|
|
1188
1127
|
const batch3 = await test_utils.fromAsync(
|
|
1189
1128
|
bucketStorage.getBucketDataBatch(
|
|
1190
1129
|
checkpoint,
|
|
1191
|
-
|
|
1130
|
+
[{ ...request, start: BigInt(batch2[0].chunkData.next_after) }],
|
|
1192
1131
|
options
|
|
1193
1132
|
)
|
|
1194
1133
|
);
|
|
@@ -1213,31 +1152,28 @@ bucket_definitions:
|
|
|
1213
1152
|
)
|
|
1214
1153
|
);
|
|
1215
1154
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1155
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1156
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1157
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1216
1158
|
|
|
1217
|
-
|
|
1218
|
-
await
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
},
|
|
1229
|
-
afterReplicaId: `test${i}`
|
|
1230
|
-
});
|
|
1231
|
-
}
|
|
1159
|
+
for (let i = 1; i <= 6; i++) {
|
|
1160
|
+
await writer.save({
|
|
1161
|
+
sourceTable,
|
|
1162
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1163
|
+
after: {
|
|
1164
|
+
id: `test${i}`,
|
|
1165
|
+
description: `test${i}`
|
|
1166
|
+
},
|
|
1167
|
+
afterReplicaId: `test${i}`
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1232
1170
|
|
|
1233
|
-
|
|
1234
|
-
});
|
|
1171
|
+
await writer.commit('1/1');
|
|
1235
1172
|
|
|
1236
1173
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1237
1174
|
|
|
1238
|
-
const
|
|
1239
|
-
|
|
1240
|
-
);
|
|
1175
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1176
|
+
const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
|
|
1241
1177
|
|
|
1242
1178
|
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
1243
1179
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
@@ -1253,13 +1189,9 @@ bucket_definitions:
|
|
|
1253
1189
|
});
|
|
1254
1190
|
|
|
1255
1191
|
const batch2 = await test_utils.oneFromAsync(
|
|
1256
|
-
bucketStorage.getBucketDataBatch(
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
{
|
|
1260
|
-
limit: 4
|
|
1261
|
-
}
|
|
1262
|
-
)
|
|
1192
|
+
bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
1193
|
+
limit: 4
|
|
1194
|
+
})
|
|
1263
1195
|
);
|
|
1264
1196
|
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
1265
1197
|
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
@@ -1273,13 +1205,9 @@ bucket_definitions:
|
|
|
1273
1205
|
});
|
|
1274
1206
|
|
|
1275
1207
|
const batch3 = await test_utils.fromAsync(
|
|
1276
|
-
bucketStorage.getBucketDataBatch(
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
{
|
|
1280
|
-
limit: 4
|
|
1281
|
-
}
|
|
1282
|
-
)
|
|
1208
|
+
bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
|
|
1209
|
+
limit: 4
|
|
1210
|
+
})
|
|
1283
1211
|
);
|
|
1284
1212
|
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
1285
1213
|
|
|
@@ -1304,48 +1232,41 @@ bucket_definitions:
|
|
|
1304
1232
|
)
|
|
1305
1233
|
);
|
|
1306
1234
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1235
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1236
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1237
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1307
1238
|
|
|
1308
|
-
|
|
1309
|
-
await
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
},
|
|
1321
|
-
afterReplicaId: `test${i}`
|
|
1322
|
-
});
|
|
1323
|
-
}
|
|
1239
|
+
for (let i = 1; i <= 10; i++) {
|
|
1240
|
+
await writer.save({
|
|
1241
|
+
sourceTable,
|
|
1242
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1243
|
+
after: {
|
|
1244
|
+
id: `test${i}`,
|
|
1245
|
+
description: `test${i}`,
|
|
1246
|
+
bucket: i == 1 ? 'global1' : 'global2'
|
|
1247
|
+
},
|
|
1248
|
+
afterReplicaId: `test${i}`
|
|
1249
|
+
});
|
|
1250
|
+
}
|
|
1324
1251
|
|
|
1325
|
-
|
|
1326
|
-
});
|
|
1252
|
+
await writer.commit('1/1');
|
|
1327
1253
|
|
|
1328
1254
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1255
|
+
const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
|
|
1256
|
+
const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
|
|
1329
1257
|
const batch = await test_utils.fromAsync(
|
|
1330
|
-
bucketStorage.getBucketDataBatch(
|
|
1331
|
-
checkpoint,
|
|
1332
|
-
bucketRequestMap(syncRules, [
|
|
1333
|
-
['global1[]', 0n],
|
|
1334
|
-
['global2[]', 0n]
|
|
1335
|
-
]),
|
|
1336
|
-
options
|
|
1337
|
-
)
|
|
1258
|
+
bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options)
|
|
1338
1259
|
);
|
|
1339
1260
|
|
|
1340
|
-
return {
|
|
1261
|
+
return { batch, global1Request, global2Request };
|
|
1341
1262
|
};
|
|
1342
1263
|
|
|
1343
1264
|
test('batch has_more (1)', async () => {
|
|
1344
|
-
const { batch,
|
|
1265
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 5 });
|
|
1345
1266
|
expect(batch.length).toEqual(2);
|
|
1346
1267
|
|
|
1347
|
-
expect(batch[0].chunkData.bucket).toEqual(
|
|
1348
|
-
expect(batch[1].chunkData.bucket).toEqual(
|
|
1268
|
+
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1269
|
+
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1349
1270
|
|
|
1350
1271
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1351
1272
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
@@ -1372,11 +1293,11 @@ bucket_definitions:
|
|
|
1372
1293
|
});
|
|
1373
1294
|
|
|
1374
1295
|
test('batch has_more (2)', async () => {
|
|
1375
|
-
const { batch,
|
|
1296
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 11 });
|
|
1376
1297
|
expect(batch.length).toEqual(2);
|
|
1377
1298
|
|
|
1378
|
-
expect(batch[0].chunkData.bucket).toEqual(
|
|
1379
|
-
expect(batch[1].chunkData.bucket).toEqual(
|
|
1299
|
+
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1300
|
+
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1380
1301
|
|
|
1381
1302
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1382
1303
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
@@ -1409,12 +1330,12 @@ bucket_definitions:
|
|
|
1409
1330
|
|
|
1410
1331
|
test('batch has_more (3)', async () => {
|
|
1411
1332
|
// 50 bytes is more than 1 row, less than 2 rows
|
|
1412
|
-
const { batch,
|
|
1333
|
+
const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 });
|
|
1413
1334
|
|
|
1414
1335
|
expect(batch.length).toEqual(3);
|
|
1415
|
-
expect(batch[0].chunkData.bucket).toEqual(
|
|
1416
|
-
expect(batch[1].chunkData.bucket).toEqual(
|
|
1417
|
-
expect(batch[2].chunkData.bucket).toEqual(
|
|
1336
|
+
expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket);
|
|
1337
|
+
expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1338
|
+
expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket);
|
|
1418
1339
|
|
|
1419
1340
|
expect(test_utils.getBatchData(batch[0])).toEqual([
|
|
1420
1341
|
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
|
|
@@ -1458,10 +1379,9 @@ bucket_definitions:
|
|
|
1458
1379
|
|
|
1459
1380
|
const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
|
|
1460
1381
|
const storage = f.getInstance(r.persisted_sync_rules!);
|
|
1461
|
-
await storage.
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
});
|
|
1382
|
+
await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1383
|
+
await writer.markAllSnapshotDone('1/0');
|
|
1384
|
+
await writer.keepalive('1/0');
|
|
1465
1385
|
|
|
1466
1386
|
await f.getStorageMetrics();
|
|
1467
1387
|
// We don't care about the specific values here
|
|
@@ -1487,36 +1407,35 @@ bucket_definitions:
|
|
|
1487
1407
|
)
|
|
1488
1408
|
);
|
|
1489
1409
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
const
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
});
|
|
1410
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1411
|
+
|
|
1412
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
|
|
1413
|
+
const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
|
|
1414
|
+
|
|
1415
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1416
|
+
// This saves a record to current_data, but not bucket_data.
|
|
1417
|
+
// This causes a checkpoint to be created without increasing the op_id sequence.
|
|
1418
|
+
await writer.save({
|
|
1419
|
+
sourceTable: sourceTableIgnore,
|
|
1420
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1421
|
+
after: {
|
|
1422
|
+
id: 'test1'
|
|
1423
|
+
},
|
|
1424
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1506
1425
|
});
|
|
1426
|
+
const result1 = await writer.flush();
|
|
1507
1427
|
|
|
1508
1428
|
const checkpoint1 = result1!.flushed_op;
|
|
1509
1429
|
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
afterReplicaId: test_utils.rid('test2')
|
|
1518
|
-
});
|
|
1430
|
+
await writer.save({
|
|
1431
|
+
sourceTable: sourceTable,
|
|
1432
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1433
|
+
after: {
|
|
1434
|
+
id: 'test2'
|
|
1435
|
+
},
|
|
1436
|
+
afterReplicaId: test_utils.rid('test2')
|
|
1519
1437
|
});
|
|
1438
|
+
const result2 = await writer.flush();
|
|
1520
1439
|
|
|
1521
1440
|
const checkpoint2 = result2!.flushed_op;
|
|
1522
1441
|
// we expect 0n and 1n, or 1n and 2n.
|
|
@@ -1539,35 +1458,27 @@ bucket_definitions:
|
|
|
1539
1458
|
)
|
|
1540
1459
|
);
|
|
1541
1460
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
await
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
});
|
|
1555
|
-
await batch.commit('1/1');
|
|
1461
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1462
|
+
|
|
1463
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1464
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1465
|
+
await writer.save({
|
|
1466
|
+
sourceTable,
|
|
1467
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1468
|
+
after: {
|
|
1469
|
+
id: 'test1',
|
|
1470
|
+
description: 'test1a'
|
|
1471
|
+
},
|
|
1472
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1556
1473
|
});
|
|
1474
|
+
await writer.commit('1/1');
|
|
1557
1475
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1558
1476
|
|
|
1559
|
-
const
|
|
1560
|
-
|
|
1561
|
-
];
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
]);
|
|
1565
|
-
const checksums2 = [
|
|
1566
|
-
...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values()
|
|
1567
|
-
];
|
|
1568
|
-
expect(checksums2).toEqual([
|
|
1569
|
-
{ bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 }
|
|
1570
|
-
]);
|
|
1477
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1478
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
1479
|
+
expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1480
|
+
const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
|
|
1481
|
+
expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1571
1482
|
});
|
|
1572
1483
|
|
|
1573
1484
|
testChecksumBatching(config);
|
|
@@ -1586,28 +1497,26 @@ bucket_definitions:
|
|
|
1586
1497
|
)
|
|
1587
1498
|
);
|
|
1588
1499
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1500
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1501
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1502
|
+
await writer.commit('1/1');
|
|
1589
1503
|
|
|
1590
|
-
await bucketStorage.
|
|
1591
|
-
|
|
1592
|
-
await batch.commit('1/1');
|
|
1593
|
-
|
|
1594
|
-
const cp1 = await bucketStorage.getCheckpoint();
|
|
1595
|
-
expect(cp1.lsn).toEqual('1/1');
|
|
1504
|
+
const cp1 = await bucketStorage.getCheckpoint();
|
|
1505
|
+
expect(cp1.lsn).toEqual('1/1');
|
|
1596
1506
|
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1507
|
+
await writer.commit('2/1', { createEmptyCheckpoints: true });
|
|
1508
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1509
|
+
expect(cp2.lsn).toEqual('2/1');
|
|
1600
1510
|
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1511
|
+
await writer.keepalive('3/1');
|
|
1512
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1513
|
+
expect(cp3.lsn).toEqual('3/1');
|
|
1604
1514
|
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
});
|
|
1515
|
+
// For the last one, we skip creating empty checkpoints
|
|
1516
|
+
// This means the LSN stays at 3/1.
|
|
1517
|
+
await writer.commit('4/1', { createEmptyCheckpoints: false });
|
|
1518
|
+
const cp4 = await bucketStorage.getCheckpoint();
|
|
1519
|
+
expect(cp4.lsn).toEqual('3/1');
|
|
1611
1520
|
});
|
|
1612
1521
|
|
|
1613
1522
|
test('empty checkpoints (2)', async () => {
|
|
@@ -1626,40 +1535,38 @@ bucket_definitions:
|
|
|
1626
1535
|
)
|
|
1627
1536
|
);
|
|
1628
1537
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1538
|
+
await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1539
|
+
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1540
|
+
const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
1629
1541
|
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
await
|
|
1633
|
-
await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch2) => {
|
|
1634
|
-
await batch1.markAllSnapshotDone('1/1');
|
|
1635
|
-
await batch1.commit('1/1');
|
|
1636
|
-
|
|
1637
|
-
await batch1.commit('2/1', { createEmptyCheckpoints: false });
|
|
1638
|
-
const cp2 = await bucketStorage.getCheckpoint();
|
|
1639
|
-
expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
|
|
1542
|
+
// We simulate two concurrent batches, but sequential calls are enough for this test.
|
|
1543
|
+
await writer1.markAllSnapshotDone('1/1');
|
|
1544
|
+
await writer1.commit('1/1');
|
|
1640
1545
|
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1546
|
+
await writer1.commit('2/1', { createEmptyCheckpoints: false });
|
|
1547
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1548
|
+
expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
|
|
1549
|
+
|
|
1550
|
+
await writer2.save({
|
|
1551
|
+
sourceTable,
|
|
1552
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1553
|
+
after: {
|
|
1554
|
+
id: 'test1',
|
|
1555
|
+
description: 'test1a'
|
|
1556
|
+
},
|
|
1557
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1558
|
+
});
|
|
1559
|
+
// This simulates what happens on a snapshot processor.
|
|
1560
|
+
// This may later change to a flush() rather than commit().
|
|
1561
|
+
await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
|
|
1653
1562
|
|
|
1654
|
-
|
|
1655
|
-
|
|
1563
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1564
|
+
expect(cp3.lsn).toEqual('1/1'); // Still unchanged
|
|
1656
1565
|
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
});
|
|
1662
|
-
});
|
|
1566
|
+
// This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
|
|
1567
|
+
await writer1.commit('4/1', { createEmptyCheckpoints: false });
|
|
1568
|
+
const cp4 = await bucketStorage.getCheckpoint();
|
|
1569
|
+
expect(cp4.lsn).toEqual('4/1');
|
|
1663
1570
|
});
|
|
1664
1571
|
|
|
1665
1572
|
test('empty checkpoints (sync rule activation)', async () => {
|
|
@@ -1679,29 +1586,24 @@ bucket_definitions:
|
|
|
1679
1586
|
);
|
|
1680
1587
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1681
1588
|
|
|
1682
|
-
await bucketStorage.
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
});
|
|
1589
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1590
|
+
const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
|
|
1591
|
+
expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1592
|
+
// Snapshot is only valid once we reach 3/1
|
|
1593
|
+
await writer.markAllSnapshotDone('3/1');
|
|
1688
1594
|
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1693
|
-
});
|
|
1595
|
+
// 2/1 < 3/1 - snapshot not valid yet, block checkpoint
|
|
1596
|
+
const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
|
|
1597
|
+
expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1694
1598
|
|
|
1695
1599
|
// No empty checkpoint should be created by the commit above.
|
|
1696
1600
|
const cp1 = await bucketStorage.getCheckpoint();
|
|
1697
1601
|
expect(cp1.lsn).toEqual(null);
|
|
1698
1602
|
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true });
|
|
1704
|
-
});
|
|
1603
|
+
// After this commit, the snapshot should be valid.
|
|
1604
|
+
// We specifically check that this is done even if createEmptyCheckpoints: false.
|
|
1605
|
+
const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
|
|
1606
|
+
expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
|
|
1705
1607
|
|
|
1706
1608
|
// Now, the checkpoint should advance the sync rules active.
|
|
1707
1609
|
const cp2 = await bucketStorage.getCheckpoint();
|
|
@@ -1710,11 +1612,9 @@ bucket_definitions:
|
|
|
1710
1612
|
const activeSyncRules = await factory.getActiveSyncRulesContent();
|
|
1711
1613
|
expect(activeSyncRules?.id).toEqual(syncRules.id);
|
|
1712
1614
|
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false });
|
|
1717
|
-
});
|
|
1615
|
+
// At this point, it should be a truely empty checkpoint
|
|
1616
|
+
const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
|
|
1617
|
+
expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
|
|
1718
1618
|
|
|
1719
1619
|
// Unchanged
|
|
1720
1620
|
const cp3 = await bucketStorage.getCheckpoint();
|
|
@@ -1737,44 +1637,46 @@ bucket_definitions:
|
|
|
1737
1637
|
)
|
|
1738
1638
|
);
|
|
1739
1639
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1640
|
+
await using snapshotWriter = await bucketStorage.createWriter({
|
|
1641
|
+
...test_utils.BATCH_OPTIONS,
|
|
1642
|
+
skipExistingRows: true
|
|
1643
|
+
});
|
|
1644
|
+
await using streamingWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1645
|
+
const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
|
|
1646
|
+
const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
|
|
1740
1647
|
|
|
1741
|
-
|
|
1742
|
-
// We simulate two concurrent batches, and nesting is the easiest way to do this.
|
|
1648
|
+
// We simulate two concurrent batches; separate writers are enough for this test.
|
|
1743
1649
|
// For this test, we assume that we start with a row "test1", which is picked up by a snapshot
|
|
1744
1650
|
// query, right before the delete is streamed. But the snapshot query is only persisted _after_
|
|
1745
1651
|
// the delete is streamed, and we need to ensure that the streamed delete takes precedence.
|
|
1746
|
-
await
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
},
|
|
1754
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
1755
|
-
});
|
|
1756
|
-
await streamingBatch.commit('2/1');
|
|
1757
|
-
|
|
1758
|
-
await snapshotBatch.save({
|
|
1759
|
-
sourceTable,
|
|
1760
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1761
|
-
after: {
|
|
1762
|
-
id: 'test1',
|
|
1763
|
-
description: 'test1a'
|
|
1764
|
-
},
|
|
1765
|
-
afterReplicaId: test_utils.rid('test1')
|
|
1766
|
-
});
|
|
1767
|
-
await snapshotBatch.markAllSnapshotDone('3/1');
|
|
1768
|
-
await snapshotBatch.commit('1/1');
|
|
1769
|
-
|
|
1770
|
-
await streamingBatch.keepalive('3/1');
|
|
1771
|
-
});
|
|
1652
|
+
await streamingWriter.save({
|
|
1653
|
+
sourceTable: streamingTable,
|
|
1654
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
1655
|
+
before: {
|
|
1656
|
+
id: 'test1'
|
|
1657
|
+
},
|
|
1658
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
1772
1659
|
});
|
|
1660
|
+
await streamingWriter.commit('2/1');
|
|
1661
|
+
|
|
1662
|
+
await snapshotWriter.save({
|
|
1663
|
+
sourceTable: snapshotTable,
|
|
1664
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1665
|
+
after: {
|
|
1666
|
+
id: 'test1',
|
|
1667
|
+
description: 'test1a'
|
|
1668
|
+
},
|
|
1669
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1670
|
+
});
|
|
1671
|
+
await snapshotWriter.markAllSnapshotDone('3/1');
|
|
1672
|
+
await snapshotWriter.commit('1/1');
|
|
1673
|
+
|
|
1674
|
+
await streamingWriter.keepalive('3/1');
|
|
1773
1675
|
|
|
1774
1676
|
const cp = await bucketStorage.getCheckpoint();
|
|
1775
1677
|
expect(cp.lsn).toEqual('3/1');
|
|
1776
1678
|
const data = await test_utils.fromAsync(
|
|
1777
|
-
bucketStorage.getBucketDataBatch(cp.checkpoint,
|
|
1679
|
+
bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')])
|
|
1778
1680
|
);
|
|
1779
1681
|
|
|
1780
1682
|
expect(data).toEqual([]);
|
|
@@ -1805,38 +1707,39 @@ bucket_definitions:
|
|
|
1805
1707
|
)
|
|
1806
1708
|
);
|
|
1807
1709
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1808
|
-
|
|
1809
|
-
const sourceTable = test_utils.
|
|
1810
|
-
await
|
|
1811
|
-
|
|
1812
|
-
for (let
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
});
|
|
1825
|
-
}
|
|
1710
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1711
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1712
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1713
|
+
for (let u of ['u1', 'u2', 'u3', 'u4']) {
|
|
1714
|
+
for (let t of ['t1', 't2', 't3', 't4']) {
|
|
1715
|
+
const id = `${t}_${u}`;
|
|
1716
|
+
await writer.save({
|
|
1717
|
+
sourceTable,
|
|
1718
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1719
|
+
after: {
|
|
1720
|
+
id,
|
|
1721
|
+
description: `${t} description`,
|
|
1722
|
+
user_id: u
|
|
1723
|
+
},
|
|
1724
|
+
afterReplicaId: test_utils.rid(id)
|
|
1725
|
+
});
|
|
1826
1726
|
}
|
|
1827
|
-
|
|
1828
|
-
|
|
1727
|
+
}
|
|
1728
|
+
await writer.commit('1/1');
|
|
1829
1729
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1830
1730
|
|
|
1831
1731
|
bucketStorage.clearChecksumCache();
|
|
1832
|
-
const
|
|
1833
|
-
const
|
|
1732
|
+
const users = ['u1', 'u2', 'u3', 'u4'];
|
|
1733
|
+
const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
|
|
1734
|
+
const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
|
|
1735
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
|
|
1834
1736
|
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1737
|
+
const expected = bucketRequests.map((request, index) => ({
|
|
1738
|
+
bucket: request.bucket,
|
|
1739
|
+
count: 4,
|
|
1740
|
+
checksum: expectedChecksums[index]
|
|
1741
|
+
}));
|
|
1742
|
+
expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1743
|
+
expect(checksums).toEqual(expected);
|
|
1841
1744
|
});
|
|
1842
1745
|
}
|