@powersync/service-core-tests 0.15.2 → 0.15.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,55 +1,3 @@
1
- var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
- if (value !== null && value !== void 0) {
3
- if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
- var dispose, inner;
5
- if (async) {
6
- if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
- dispose = value[Symbol.asyncDispose];
8
- }
9
- if (dispose === void 0) {
10
- if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
- dispose = value[Symbol.dispose];
12
- if (async) inner = dispose;
13
- }
14
- if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
- if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
- env.stack.push({ value: value, dispose: dispose, async: async });
17
- }
18
- else if (async) {
19
- env.stack.push({ async: true });
20
- }
21
- return value;
22
- };
23
- var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
- return function (env) {
25
- function fail(e) {
26
- env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
- env.hasError = true;
28
- }
29
- var r, s = 0;
30
- function next() {
31
- while (r = env.stack.pop()) {
32
- try {
33
- if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
- if (r.dispose) {
35
- var result = r.dispose.call(r.value);
36
- if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
- }
38
- else s |= 1;
39
- }
40
- catch (e) {
41
- fail(e);
42
- }
43
- }
44
- if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
- if (env.hasError) throw env.error;
46
- }
47
- return next();
48
- };
49
- })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
- var e = new Error(message);
51
- return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
- });
53
1
  import { CURRENT_STORAGE_VERSION, getUuidReplicaIdentityBson, reduceBucket, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
54
2
  import { describe, expect, test } from 'vitest';
55
3
  import * as test_utils from '../test-utils/test-utils-index.js';
@@ -78,372 +26,223 @@ export function registerDataStorageDataTests(config) {
78
26
  const generateStorageFactory = config.factory;
79
27
  const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
80
28
  test('removing row', async () => {
81
- const env_1 = { stack: [], error: void 0, hasError: false };
82
- try {
83
- const factory = __addDisposableResource(env_1, await generateStorageFactory(), true);
84
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
29
+ await using factory = await generateStorageFactory();
30
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
85
31
  bucket_definitions:
86
32
  global:
87
33
  data:
88
34
  - SELECT id, description FROM "%"
89
35
  `, { storageVersion }));
90
- const bucketStorage = factory.getInstance(syncRules);
91
- const writer = __addDisposableResource(env_1, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
92
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
93
- await writer.markAllSnapshotDone('1/1');
94
- await writer.save({
95
- sourceTable: testTable,
96
- tag: storage.SaveOperationTag.INSERT,
97
- after: {
98
- id: 'test1',
99
- description: 'test1'
100
- },
101
- afterReplicaId: test_utils.rid('test1')
102
- });
103
- await writer.save({
104
- sourceTable: testTable,
105
- tag: storage.SaveOperationTag.DELETE,
106
- beforeReplicaId: test_utils.rid('test1')
107
- });
108
- await writer.commit('1/1');
109
- const { checkpoint } = await bucketStorage.getCheckpoint();
110
- const request = bucketRequest(syncRules, 'global[]');
111
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
112
- const data = batch[0].chunkData.data.map((d) => {
113
- return {
114
- op: d.op,
115
- object_id: d.object_id,
116
- checksum: d.checksum
117
- };
118
- });
119
- const c1 = 2871785649;
120
- const c2 = 2872534815;
121
- expect(data).toEqual([
122
- { op: 'PUT', object_id: 'test1', checksum: c1 },
123
- { op: 'REMOVE', object_id: 'test1', checksum: c2 }
124
- ]);
125
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
126
- expect(checksums).toEqual([
127
- {
128
- bucket: request.bucket,
129
- checksum: (c1 + c2) & 0xffffffff,
130
- count: 2
131
- }
132
- ]);
133
- }
134
- catch (e_1) {
135
- env_1.error = e_1;
136
- env_1.hasError = true;
137
- }
138
- finally {
139
- const result_1 = __disposeResources(env_1);
140
- if (result_1)
141
- await result_1;
142
- }
36
+ const bucketStorage = factory.getInstance(syncRules);
37
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
38
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
39
+ await writer.markAllSnapshotDone('1/1');
40
+ await writer.save({
41
+ sourceTable: testTable,
42
+ tag: storage.SaveOperationTag.INSERT,
43
+ after: {
44
+ id: 'test1',
45
+ description: 'test1'
46
+ },
47
+ afterReplicaId: test_utils.rid('test1')
48
+ });
49
+ await writer.save({
50
+ sourceTable: testTable,
51
+ tag: storage.SaveOperationTag.DELETE,
52
+ beforeReplicaId: test_utils.rid('test1')
53
+ });
54
+ await writer.commit('1/1');
55
+ const { checkpoint } = await bucketStorage.getCheckpoint();
56
+ const request = bucketRequest(syncRules, 'global[]');
57
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
58
+ const data = batch[0].chunkData.data.map((d) => {
59
+ return {
60
+ op: d.op,
61
+ object_id: d.object_id,
62
+ checksum: d.checksum
63
+ };
64
+ });
65
+ const c1 = 2871785649;
66
+ const c2 = 2872534815;
67
+ expect(data).toEqual([
68
+ { op: 'PUT', object_id: 'test1', checksum: c1 },
69
+ { op: 'REMOVE', object_id: 'test1', checksum: c2 }
70
+ ]);
71
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
72
+ expect(checksums).toEqual([
73
+ {
74
+ bucket: request.bucket,
75
+ checksum: (c1 + c2) & 0xffffffff,
76
+ count: 2
77
+ }
78
+ ]);
143
79
  });
144
80
  test('insert after delete in new batch', async () => {
145
- const env_2 = { stack: [], error: void 0, hasError: false };
146
- try {
147
- const factory = __addDisposableResource(env_2, await generateStorageFactory(), true);
148
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
81
+ await using factory = await generateStorageFactory();
82
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
149
83
  bucket_definitions:
150
84
  global:
151
85
  data:
152
86
  - SELECT id, description FROM "%"
153
87
  `, { storageVersion }));
154
- const bucketStorage = factory.getInstance(syncRules);
155
- const writer = __addDisposableResource(env_2, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
156
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
157
- await writer.markAllSnapshotDone('1/1');
158
- await writer.save({
159
- sourceTable: testTable,
160
- tag: storage.SaveOperationTag.DELETE,
161
- beforeReplicaId: test_utils.rid('test1')
162
- });
163
- await writer.commit('0/1');
164
- await writer.save({
165
- sourceTable: testTable,
166
- tag: storage.SaveOperationTag.INSERT,
167
- after: {
168
- id: 'test1',
169
- description: 'test1'
170
- },
171
- afterReplicaId: test_utils.rid('test1')
172
- });
173
- await writer.commit('2/1');
174
- const { checkpoint } = await bucketStorage.getCheckpoint();
175
- const request = bucketRequest(syncRules, 'global[]');
176
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
177
- const data = batch[0].chunkData.data.map((d) => {
178
- return {
179
- op: d.op,
180
- object_id: d.object_id,
181
- checksum: d.checksum
182
- };
183
- });
184
- const c1 = 2871785649;
185
- expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
186
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
187
- expect(checksums).toEqual([
188
- {
189
- bucket: request.bucket,
190
- checksum: c1 & 0xffffffff,
191
- count: 1
192
- }
193
- ]);
194
- }
195
- catch (e_2) {
196
- env_2.error = e_2;
197
- env_2.hasError = true;
198
- }
199
- finally {
200
- const result_2 = __disposeResources(env_2);
201
- if (result_2)
202
- await result_2;
203
- }
88
+ const bucketStorage = factory.getInstance(syncRules);
89
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
90
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
91
+ await writer.markAllSnapshotDone('1/1');
92
+ await writer.save({
93
+ sourceTable: testTable,
94
+ tag: storage.SaveOperationTag.DELETE,
95
+ beforeReplicaId: test_utils.rid('test1')
96
+ });
97
+ await writer.commit('0/1');
98
+ await writer.save({
99
+ sourceTable: testTable,
100
+ tag: storage.SaveOperationTag.INSERT,
101
+ after: {
102
+ id: 'test1',
103
+ description: 'test1'
104
+ },
105
+ afterReplicaId: test_utils.rid('test1')
106
+ });
107
+ await writer.commit('2/1');
108
+ const { checkpoint } = await bucketStorage.getCheckpoint();
109
+ const request = bucketRequest(syncRules, 'global[]');
110
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
111
+ const data = batch[0].chunkData.data.map((d) => {
112
+ return {
113
+ op: d.op,
114
+ object_id: d.object_id,
115
+ checksum: d.checksum
116
+ };
117
+ });
118
+ const c1 = 2871785649;
119
+ expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
120
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
121
+ expect(checksums).toEqual([
122
+ {
123
+ bucket: request.bucket,
124
+ checksum: c1 & 0xffffffff,
125
+ count: 1
126
+ }
127
+ ]);
204
128
  });
205
129
  test('update after delete in new batch', async () => {
206
- const env_3 = { stack: [], error: void 0, hasError: false };
207
- try {
208
- // Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way.
209
- const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
210
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
130
+ // Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way.
131
+ await using factory = await generateStorageFactory();
132
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
211
133
  bucket_definitions:
212
134
  global:
213
135
  data:
214
136
  - SELECT id, description FROM "%"
215
137
  `, { storageVersion }));
216
- const bucketStorage = factory.getInstance(syncRules);
217
- const writer = __addDisposableResource(env_3, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
218
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
219
- await writer.markAllSnapshotDone('1/1');
220
- await writer.save({
221
- sourceTable: testTable,
222
- tag: storage.SaveOperationTag.DELETE,
223
- beforeReplicaId: test_utils.rid('test1')
224
- });
225
- await writer.commit('0/1');
226
- await writer.save({
227
- sourceTable: testTable,
228
- tag: storage.SaveOperationTag.UPDATE,
229
- before: {
230
- id: 'test1'
231
- },
232
- after: {
233
- id: 'test1',
234
- description: 'test1'
235
- },
236
- beforeReplicaId: test_utils.rid('test1'),
237
- afterReplicaId: test_utils.rid('test1')
238
- });
239
- await writer.commit('2/1');
240
- const { checkpoint } = await bucketStorage.getCheckpoint();
241
- const request = bucketRequest(syncRules, 'global[]');
242
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
243
- const data = batch[0].chunkData.data.map((d) => {
244
- return {
245
- op: d.op,
246
- object_id: d.object_id,
247
- checksum: d.checksum
248
- };
249
- });
250
- const c1 = 2871785649;
251
- expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
252
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
253
- expect(checksums).toEqual([
254
- {
255
- bucket: request.bucket,
256
- checksum: c1 & 0xffffffff,
257
- count: 1
258
- }
259
- ]);
260
- }
261
- catch (e_3) {
262
- env_3.error = e_3;
263
- env_3.hasError = true;
264
- }
265
- finally {
266
- const result_3 = __disposeResources(env_3);
267
- if (result_3)
268
- await result_3;
269
- }
138
+ const bucketStorage = factory.getInstance(syncRules);
139
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
140
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
141
+ await writer.markAllSnapshotDone('1/1');
142
+ await writer.save({
143
+ sourceTable: testTable,
144
+ tag: storage.SaveOperationTag.DELETE,
145
+ beforeReplicaId: test_utils.rid('test1')
146
+ });
147
+ await writer.commit('0/1');
148
+ await writer.save({
149
+ sourceTable: testTable,
150
+ tag: storage.SaveOperationTag.UPDATE,
151
+ before: {
152
+ id: 'test1'
153
+ },
154
+ after: {
155
+ id: 'test1',
156
+ description: 'test1'
157
+ },
158
+ beforeReplicaId: test_utils.rid('test1'),
159
+ afterReplicaId: test_utils.rid('test1')
160
+ });
161
+ await writer.commit('2/1');
162
+ const { checkpoint } = await bucketStorage.getCheckpoint();
163
+ const request = bucketRequest(syncRules, 'global[]');
164
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
165
+ const data = batch[0].chunkData.data.map((d) => {
166
+ return {
167
+ op: d.op,
168
+ object_id: d.object_id,
169
+ checksum: d.checksum
170
+ };
171
+ });
172
+ const c1 = 2871785649;
173
+ expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
174
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
175
+ expect(checksums).toEqual([
176
+ {
177
+ bucket: request.bucket,
178
+ checksum: c1 & 0xffffffff,
179
+ count: 1
180
+ }
181
+ ]);
270
182
  });
271
183
  test('insert after delete in same batch', async () => {
272
- const env_4 = { stack: [], error: void 0, hasError: false };
273
- try {
274
- const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
275
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
184
+ await using factory = await generateStorageFactory();
185
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
276
186
  bucket_definitions:
277
187
  global:
278
188
  data:
279
189
  - SELECT id, description FROM "%"
280
190
  `, {
281
- storageVersion
282
- }));
283
- const bucketStorage = factory.getInstance(syncRules);
284
- const writer = __addDisposableResource(env_4, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
285
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
286
- await writer.markAllSnapshotDone('1/1');
287
- await writer.save({
288
- sourceTable: testTable,
289
- tag: storage.SaveOperationTag.DELETE,
290
- beforeReplicaId: test_utils.rid('test1')
291
- });
292
- await writer.save({
293
- sourceTable: testTable,
294
- tag: storage.SaveOperationTag.INSERT,
295
- after: {
296
- id: 'test1',
297
- description: 'test1'
298
- },
299
- afterReplicaId: test_utils.rid('test1')
300
- });
301
- await writer.commit('1/1');
302
- const { checkpoint } = await bucketStorage.getCheckpoint();
303
- const request = bucketRequest(syncRules, 'global[]');
304
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
305
- const data = batch[0].chunkData.data.map((d) => {
306
- return {
307
- op: d.op,
308
- object_id: d.object_id,
309
- checksum: d.checksum
310
- };
311
- });
312
- const c1 = 2871785649;
313
- expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
314
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
315
- expect(checksums).toEqual([
316
- {
317
- bucket: request.bucket,
318
- checksum: c1 & 0xffffffff,
319
- count: 1
320
- }
321
- ]);
322
- }
323
- catch (e_4) {
324
- env_4.error = e_4;
325
- env_4.hasError = true;
326
- }
327
- finally {
328
- const result_4 = __disposeResources(env_4);
329
- if (result_4)
330
- await result_4;
331
- }
332
- });
333
- test('(insert, delete, insert), (delete)', async () => {
334
- const env_5 = { stack: [], error: void 0, hasError: false };
335
- try {
336
- const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
337
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
338
- bucket_definitions:
339
- global:
340
- data:
341
- - SELECT id, description FROM "%"
342
- `, {
343
- storageVersion
344
- }));
345
- const bucketStorage = factory.getInstance(syncRules);
346
- {
347
- const env_6 = { stack: [], error: void 0, hasError: false };
348
- try {
349
- const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
350
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
351
- await writer.markAllSnapshotDone('1/1');
352
- await writer.save({
353
- sourceTable,
354
- tag: storage.SaveOperationTag.INSERT,
355
- after: {
356
- id: 'test1',
357
- description: 'test1'
358
- },
359
- afterReplicaId: test_utils.rid('test1')
360
- });
361
- await writer.save({
362
- sourceTable,
363
- tag: storage.SaveOperationTag.DELETE,
364
- beforeReplicaId: test_utils.rid('test1')
365
- });
366
- await writer.save({
367
- sourceTable,
368
- tag: storage.SaveOperationTag.INSERT,
369
- after: {
370
- id: 'test1',
371
- description: 'test1'
372
- },
373
- afterReplicaId: test_utils.rid('test1')
374
- });
375
- await writer.commit('1/1');
376
- }
377
- catch (e_5) {
378
- env_6.error = e_5;
379
- env_6.hasError = true;
380
- }
381
- finally {
382
- const result_5 = __disposeResources(env_6);
383
- if (result_5)
384
- await result_5;
385
- }
386
- }
191
+ storageVersion
192
+ }));
193
+ const bucketStorage = factory.getInstance(syncRules);
194
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
195
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
196
+ await writer.markAllSnapshotDone('1/1');
197
+ await writer.save({
198
+ sourceTable: testTable,
199
+ tag: storage.SaveOperationTag.DELETE,
200
+ beforeReplicaId: test_utils.rid('test1')
201
+ });
202
+ await writer.save({
203
+ sourceTable: testTable,
204
+ tag: storage.SaveOperationTag.INSERT,
205
+ after: {
206
+ id: 'test1',
207
+ description: 'test1'
208
+ },
209
+ afterReplicaId: test_utils.rid('test1')
210
+ });
211
+ await writer.commit('1/1');
212
+ const { checkpoint } = await bucketStorage.getCheckpoint();
213
+ const request = bucketRequest(syncRules, 'global[]');
214
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
215
+ const data = batch[0].chunkData.data.map((d) => {
216
+ return {
217
+ op: d.op,
218
+ object_id: d.object_id,
219
+ checksum: d.checksum
220
+ };
221
+ });
222
+ const c1 = 2871785649;
223
+ expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
224
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
225
+ expect(checksums).toEqual([
387
226
  {
388
- const env_7 = { stack: [], error: void 0, hasError: false };
389
- try {
390
- const writer = __addDisposableResource(env_7, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
391
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
392
- await writer.markAllSnapshotDone('1/1');
393
- await writer.save({
394
- sourceTable,
395
- tag: storage.SaveOperationTag.DELETE,
396
- beforeReplicaId: test_utils.rid('test1')
397
- });
398
- await writer.commit('2/1');
399
- }
400
- catch (e_6) {
401
- env_7.error = e_6;
402
- env_7.hasError = true;
403
- }
404
- finally {
405
- const result_6 = __disposeResources(env_7);
406
- if (result_6)
407
- await result_6;
408
- }
227
+ bucket: request.bucket,
228
+ checksum: c1 & 0xffffffff,
229
+ count: 1
409
230
  }
410
- const { checkpoint } = await bucketStorage.getCheckpoint();
411
- const request = bucketRequest(syncRules, 'global[]');
412
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
413
- expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
414
- const data = batch[0].chunkData.data.map((d) => {
415
- return {
416
- op: d.op,
417
- object_id: d.object_id,
418
- checksum: d.checksum
419
- };
420
- });
421
- expect(data).toMatchSnapshot();
422
- }
423
- catch (e_7) {
424
- env_5.error = e_7;
425
- env_5.hasError = true;
426
- }
427
- finally {
428
- const result_7 = __disposeResources(env_5);
429
- if (result_7)
430
- await result_7;
431
- }
231
+ ]);
432
232
  });
433
- test('changing client ids', async () => {
434
- const env_8 = { stack: [], error: void 0, hasError: false };
435
- try {
436
- const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
437
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
233
+ test('(insert, delete, insert), (delete)', async () => {
234
+ await using factory = await generateStorageFactory();
235
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
438
236
  bucket_definitions:
439
237
  global:
440
238
  data:
441
- - SELECT client_id as id, description FROM "%"
239
+ - SELECT id, description FROM "%"
442
240
  `, {
443
- storageVersion
444
- }));
445
- const bucketStorage = factory.getInstance(syncRules);
446
- const writer = __addDisposableResource(env_8, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
241
+ storageVersion
242
+ }));
243
+ const bucketStorage = factory.getInstance(syncRules);
244
+ {
245
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
447
246
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
448
247
  await writer.markAllSnapshotDone('1/1');
449
248
  await writer.save({
@@ -451,73 +250,15 @@ bucket_definitions:
451
250
  tag: storage.SaveOperationTag.INSERT,
452
251
  after: {
453
252
  id: 'test1',
454
- client_id: 'client1a',
455
- description: 'test1a'
456
- },
457
- afterReplicaId: test_utils.rid('test1')
458
- });
459
- await writer.save({
460
- sourceTable,
461
- tag: storage.SaveOperationTag.UPDATE,
462
- after: {
463
- id: 'test1',
464
- client_id: 'client1b',
465
- description: 'test1b'
253
+ description: 'test1'
466
254
  },
467
255
  afterReplicaId: test_utils.rid('test1')
468
256
  });
469
257
  await writer.save({
470
258
  sourceTable,
471
- tag: storage.SaveOperationTag.INSERT,
472
- after: {
473
- id: 'test2',
474
- client_id: 'client2',
475
- description: 'test2'
476
- },
477
- afterReplicaId: test_utils.rid('test2')
478
- });
479
- await writer.commit('1/1');
480
- const { checkpoint } = await bucketStorage.getCheckpoint();
481
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
482
- const data = batch[0].chunkData.data.map((d) => {
483
- return {
484
- op: d.op,
485
- object_id: d.object_id
486
- };
259
+ tag: storage.SaveOperationTag.DELETE,
260
+ beforeReplicaId: test_utils.rid('test1')
487
261
  });
488
- expect(data).toEqual([
489
- { op: 'PUT', object_id: 'client1a' },
490
- { op: 'PUT', object_id: 'client1b' },
491
- { op: 'REMOVE', object_id: 'client1a' },
492
- { op: 'PUT', object_id: 'client2' }
493
- ]);
494
- }
495
- catch (e_8) {
496
- env_8.error = e_8;
497
- env_8.hasError = true;
498
- }
499
- finally {
500
- const result_8 = __disposeResources(env_8);
501
- if (result_8)
502
- await result_8;
503
- }
504
- });
505
- test('re-apply delete', async () => {
506
- const env_9 = { stack: [], error: void 0, hasError: false };
507
- try {
508
- const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
509
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
510
- bucket_definitions:
511
- global:
512
- data:
513
- - SELECT id, description FROM "%"
514
- `, {
515
- storageVersion
516
- }));
517
- const bucketStorage = factory.getInstance(syncRules);
518
- const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
519
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
520
- await writer.markAllSnapshotDone('1/1');
521
262
  await writer.save({
522
263
  sourceTable,
523
264
  tag: storage.SaveOperationTag.INSERT,
@@ -527,738 +268,753 @@ bucket_definitions:
527
268
  },
528
269
  afterReplicaId: test_utils.rid('test1')
529
270
  });
530
- await writer.flush();
531
- await writer.save({
532
- sourceTable,
533
- tag: storage.SaveOperationTag.DELETE,
534
- beforeReplicaId: test_utils.rid('test1')
535
- });
536
271
  await writer.commit('1/1');
272
+ }
273
+ {
274
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
275
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
276
+ await writer.markAllSnapshotDone('1/1');
537
277
  await writer.save({
538
278
  sourceTable,
539
279
  tag: storage.SaveOperationTag.DELETE,
540
280
  beforeReplicaId: test_utils.rid('test1')
541
281
  });
542
- await writer.flush();
543
- const { checkpoint } = await bucketStorage.getCheckpoint();
544
- const request = bucketRequest(syncRules, 'global[]');
545
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
546
- const data = batch[0].chunkData.data.map((d) => {
547
- return {
548
- op: d.op,
549
- object_id: d.object_id,
550
- checksum: d.checksum
551
- };
552
- });
553
- const c1 = 2871785649;
554
- const c2 = 2872534815;
555
- expect(data).toEqual([
556
- { op: 'PUT', object_id: 'test1', checksum: c1 },
557
- { op: 'REMOVE', object_id: 'test1', checksum: c2 }
558
- ]);
559
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
560
- expect(checksums).toEqual([
561
- {
562
- bucket: bucketRequest(syncRules, 'global[]').bucket,
563
- checksum: (c1 + c2) & 0xffffffff,
564
- count: 2
565
- }
566
- ]);
567
- }
568
- catch (e_9) {
569
- env_9.error = e_9;
570
- env_9.hasError = true;
571
- }
572
- finally {
573
- const result_9 = __disposeResources(env_9);
574
- if (result_9)
575
- await result_9;
282
+ await writer.commit('2/1');
576
283
  }
284
+ const { checkpoint } = await bucketStorage.getCheckpoint();
285
+ const request = bucketRequest(syncRules, 'global[]');
286
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
287
+ expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
288
+ const data = batch[0].chunkData.data.map((d) => {
289
+ return {
290
+ op: d.op,
291
+ object_id: d.object_id,
292
+ checksum: d.checksum
293
+ };
294
+ });
295
+ expect(data).toMatchSnapshot();
296
+ });
297
+ test('changing client ids', async () => {
298
+ await using factory = await generateStorageFactory();
299
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
300
+ bucket_definitions:
301
+ global:
302
+ data:
303
+ - SELECT client_id as id, description FROM "%"
304
+ `, {
305
+ storageVersion
306
+ }));
307
+ const bucketStorage = factory.getInstance(syncRules);
308
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
309
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
310
+ await writer.markAllSnapshotDone('1/1');
311
+ await writer.save({
312
+ sourceTable,
313
+ tag: storage.SaveOperationTag.INSERT,
314
+ after: {
315
+ id: 'test1',
316
+ client_id: 'client1a',
317
+ description: 'test1a'
318
+ },
319
+ afterReplicaId: test_utils.rid('test1')
320
+ });
321
+ await writer.save({
322
+ sourceTable,
323
+ tag: storage.SaveOperationTag.UPDATE,
324
+ after: {
325
+ id: 'test1',
326
+ client_id: 'client1b',
327
+ description: 'test1b'
328
+ },
329
+ afterReplicaId: test_utils.rid('test1')
330
+ });
331
+ await writer.save({
332
+ sourceTable,
333
+ tag: storage.SaveOperationTag.INSERT,
334
+ after: {
335
+ id: 'test2',
336
+ client_id: 'client2',
337
+ description: 'test2'
338
+ },
339
+ afterReplicaId: test_utils.rid('test2')
340
+ });
341
+ await writer.commit('1/1');
342
+ const { checkpoint } = await bucketStorage.getCheckpoint();
343
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
344
+ const data = batch[0].chunkData.data.map((d) => {
345
+ return {
346
+ op: d.op,
347
+ object_id: d.object_id
348
+ };
349
+ });
350
+ expect(data).toEqual([
351
+ { op: 'PUT', object_id: 'client1a' },
352
+ { op: 'PUT', object_id: 'client1b' },
353
+ { op: 'REMOVE', object_id: 'client1a' },
354
+ { op: 'PUT', object_id: 'client2' }
355
+ ]);
356
+ });
357
+ test('re-apply delete', async () => {
358
+ await using factory = await generateStorageFactory();
359
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
360
+ bucket_definitions:
361
+ global:
362
+ data:
363
+ - SELECT id, description FROM "%"
364
+ `, {
365
+ storageVersion
366
+ }));
367
+ const bucketStorage = factory.getInstance(syncRules);
368
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
369
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
370
+ await writer.markAllSnapshotDone('1/1');
371
+ await writer.save({
372
+ sourceTable,
373
+ tag: storage.SaveOperationTag.INSERT,
374
+ after: {
375
+ id: 'test1',
376
+ description: 'test1'
377
+ },
378
+ afterReplicaId: test_utils.rid('test1')
379
+ });
380
+ await writer.flush();
381
+ await writer.save({
382
+ sourceTable,
383
+ tag: storage.SaveOperationTag.DELETE,
384
+ beforeReplicaId: test_utils.rid('test1')
385
+ });
386
+ await writer.commit('1/1');
387
+ await writer.save({
388
+ sourceTable,
389
+ tag: storage.SaveOperationTag.DELETE,
390
+ beforeReplicaId: test_utils.rid('test1')
391
+ });
392
+ await writer.flush();
393
+ const { checkpoint } = await bucketStorage.getCheckpoint();
394
+ const request = bucketRequest(syncRules, 'global[]');
395
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
396
+ const data = batch[0].chunkData.data.map((d) => {
397
+ return {
398
+ op: d.op,
399
+ object_id: d.object_id,
400
+ checksum: d.checksum
401
+ };
402
+ });
403
+ const c1 = 2871785649;
404
+ const c2 = 2872534815;
405
+ expect(data).toEqual([
406
+ { op: 'PUT', object_id: 'test1', checksum: c1 },
407
+ { op: 'REMOVE', object_id: 'test1', checksum: c2 }
408
+ ]);
409
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
410
+ expect(checksums).toEqual([
411
+ {
412
+ bucket: bucketRequest(syncRules, 'global[]').bucket,
413
+ checksum: (c1 + c2) & 0xffffffff,
414
+ count: 2
415
+ }
416
+ ]);
577
417
  });
578
418
  test('re-apply update + delete', async () => {
579
- const env_10 = { stack: [], error: void 0, hasError: false };
580
- try {
581
- const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
582
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
419
+ await using factory = await generateStorageFactory();
420
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
583
421
  bucket_definitions:
584
422
  global:
585
423
  data:
586
424
  - SELECT id, description FROM "%"
587
425
  `, { storageVersion }));
588
- const bucketStorage = factory.getInstance(syncRules);
589
- const writer = __addDisposableResource(env_10, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
590
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
591
- await writer.markAllSnapshotDone('1/1');
592
- await writer.save({
593
- sourceTable,
594
- tag: storage.SaveOperationTag.INSERT,
595
- after: {
596
- id: 'test1',
597
- description: 'test1'
598
- },
599
- afterReplicaId: test_utils.rid('test1')
600
- });
601
- await writer.flush();
602
- await writer.markAllSnapshotDone('1/1');
603
- await writer.save({
604
- sourceTable,
605
- tag: storage.SaveOperationTag.UPDATE,
606
- after: {
607
- id: 'test1',
608
- description: undefined
609
- },
610
- afterReplicaId: test_utils.rid('test1')
611
- });
612
- await writer.save({
613
- sourceTable,
614
- tag: storage.SaveOperationTag.UPDATE,
615
- after: {
616
- id: 'test1',
617
- description: undefined
618
- },
619
- afterReplicaId: test_utils.rid('test1')
620
- });
621
- await writer.save({
622
- sourceTable,
623
- tag: storage.SaveOperationTag.DELETE,
624
- beforeReplicaId: test_utils.rid('test1')
625
- });
626
- await writer.commit('1/1');
627
- await writer.markAllSnapshotDone('1/1');
628
- await writer.save({
629
- sourceTable,
630
- tag: storage.SaveOperationTag.UPDATE,
631
- after: {
632
- id: 'test1',
633
- description: undefined
634
- },
635
- afterReplicaId: test_utils.rid('test1')
636
- });
637
- await writer.save({
638
- sourceTable,
639
- tag: storage.SaveOperationTag.UPDATE,
640
- after: {
641
- id: 'test1',
642
- description: undefined
643
- },
644
- afterReplicaId: test_utils.rid('test1')
645
- });
646
- await writer.save({
647
- sourceTable,
648
- tag: storage.SaveOperationTag.DELETE,
649
- beforeReplicaId: test_utils.rid('test1')
650
- });
651
- await writer.commit('2/1');
652
- const { checkpoint } = await bucketStorage.getCheckpoint();
653
- const request = bucketRequest(syncRules, 'global[]');
654
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
655
- const data = batch[0].chunkData.data.map((d) => {
656
- return {
657
- op: d.op,
658
- object_id: d.object_id,
659
- checksum: d.checksum
660
- };
661
- });
662
- const c1 = 2871785649;
663
- const c2 = 2872534815;
664
- expect(data).toEqual([
665
- { op: 'PUT', object_id: 'test1', checksum: c1 },
666
- { op: 'PUT', object_id: 'test1', checksum: c1 },
667
- { op: 'PUT', object_id: 'test1', checksum: c1 },
668
- { op: 'REMOVE', object_id: 'test1', checksum: c2 }
669
- ]);
670
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
671
- expect(checksums).toEqual([
672
- {
673
- bucket: bucketRequest(syncRules, 'global[]').bucket,
674
- checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
675
- count: 4
676
- }
677
- ]);
678
- }
679
- catch (e_10) {
680
- env_10.error = e_10;
681
- env_10.hasError = true;
682
- }
683
- finally {
684
- const result_10 = __disposeResources(env_10);
685
- if (result_10)
686
- await result_10;
687
- }
426
+ const bucketStorage = factory.getInstance(syncRules);
427
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
428
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
429
+ await writer.markAllSnapshotDone('1/1');
430
+ await writer.save({
431
+ sourceTable,
432
+ tag: storage.SaveOperationTag.INSERT,
433
+ after: {
434
+ id: 'test1',
435
+ description: 'test1'
436
+ },
437
+ afterReplicaId: test_utils.rid('test1')
438
+ });
439
+ await writer.flush();
440
+ await writer.markAllSnapshotDone('1/1');
441
+ await writer.save({
442
+ sourceTable,
443
+ tag: storage.SaveOperationTag.UPDATE,
444
+ after: {
445
+ id: 'test1',
446
+ description: undefined
447
+ },
448
+ afterReplicaId: test_utils.rid('test1')
449
+ });
450
+ await writer.save({
451
+ sourceTable,
452
+ tag: storage.SaveOperationTag.UPDATE,
453
+ after: {
454
+ id: 'test1',
455
+ description: undefined
456
+ },
457
+ afterReplicaId: test_utils.rid('test1')
458
+ });
459
+ await writer.save({
460
+ sourceTable,
461
+ tag: storage.SaveOperationTag.DELETE,
462
+ beforeReplicaId: test_utils.rid('test1')
463
+ });
464
+ await writer.commit('1/1');
465
+ await writer.markAllSnapshotDone('1/1');
466
+ await writer.save({
467
+ sourceTable,
468
+ tag: storage.SaveOperationTag.UPDATE,
469
+ after: {
470
+ id: 'test1',
471
+ description: undefined
472
+ },
473
+ afterReplicaId: test_utils.rid('test1')
474
+ });
475
+ await writer.save({
476
+ sourceTable,
477
+ tag: storage.SaveOperationTag.UPDATE,
478
+ after: {
479
+ id: 'test1',
480
+ description: undefined
481
+ },
482
+ afterReplicaId: test_utils.rid('test1')
483
+ });
484
+ await writer.save({
485
+ sourceTable,
486
+ tag: storage.SaveOperationTag.DELETE,
487
+ beforeReplicaId: test_utils.rid('test1')
488
+ });
489
+ await writer.commit('2/1');
490
+ const { checkpoint } = await bucketStorage.getCheckpoint();
491
+ const request = bucketRequest(syncRules, 'global[]');
492
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
493
+ const data = batch[0].chunkData.data.map((d) => {
494
+ return {
495
+ op: d.op,
496
+ object_id: d.object_id,
497
+ checksum: d.checksum
498
+ };
499
+ });
500
+ const c1 = 2871785649;
501
+ const c2 = 2872534815;
502
+ expect(data).toEqual([
503
+ { op: 'PUT', object_id: 'test1', checksum: c1 },
504
+ { op: 'PUT', object_id: 'test1', checksum: c1 },
505
+ { op: 'PUT', object_id: 'test1', checksum: c1 },
506
+ { op: 'REMOVE', object_id: 'test1', checksum: c2 }
507
+ ]);
508
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
509
+ expect(checksums).toEqual([
510
+ {
511
+ bucket: bucketRequest(syncRules, 'global[]').bucket,
512
+ checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
513
+ count: 4
514
+ }
515
+ ]);
688
516
  });
689
517
  test('batch with overlapping replica ids', async () => {
690
- const env_11 = { stack: [], error: void 0, hasError: false };
691
- try {
692
- // This test checks that we get the correct output when processing rows with:
693
- // 1. changing replica ids
694
- // 2. overlapping with replica ids of other rows in the same transaction (at different times)
695
- // If operations are not processing in input order, this breaks easily.
696
- // It can break at two places:
697
- // 1. Not getting the correct "current_data" state for each operation.
698
- // 2. Output order not being correct.
699
- const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
700
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
518
+ // This test checks that we get the correct output when processing rows with:
519
+ // 1. changing replica ids
520
+ // 2. overlapping with replica ids of other rows in the same transaction (at different times)
521
+ // If operations are not processing in input order, this breaks easily.
522
+ // It can break at two places:
523
+ // 1. Not getting the correct "current_data" state for each operation.
524
+ // 2. Output order not being correct.
525
+ await using factory = await generateStorageFactory();
526
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
701
527
  bucket_definitions:
702
528
  global:
703
529
  data:
704
530
  - SELECT id, description FROM "test"
705
531
  `, { storageVersion }));
706
- const bucketStorage = factory.getInstance(syncRules);
707
- const writer = __addDisposableResource(env_11, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
708
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
709
- // Pre-setup
710
- await writer.markAllSnapshotDone('1/1');
711
- await writer.save({
712
- sourceTable,
713
- tag: storage.SaveOperationTag.INSERT,
714
- after: {
715
- id: 'test1',
716
- description: 'test1a'
717
- },
718
- afterReplicaId: test_utils.rid('test1')
719
- });
720
- await writer.save({
721
- sourceTable,
722
- tag: storage.SaveOperationTag.INSERT,
723
- after: {
724
- id: 'test2',
725
- description: 'test2a'
726
- },
727
- afterReplicaId: test_utils.rid('test2')
728
- });
729
- const result1 = await writer.flush();
730
- const checkpoint1 = result1?.flushed_op ?? 0n;
731
- // Test batch
532
+ const bucketStorage = factory.getInstance(syncRules);
533
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
534
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
535
+ // Pre-setup
536
+ await writer.markAllSnapshotDone('1/1');
537
+ await writer.save({
538
+ sourceTable,
539
+ tag: storage.SaveOperationTag.INSERT,
540
+ after: {
541
+ id: 'test1',
542
+ description: 'test1a'
543
+ },
544
+ afterReplicaId: test_utils.rid('test1')
545
+ });
546
+ await writer.save({
547
+ sourceTable,
548
+ tag: storage.SaveOperationTag.INSERT,
549
+ after: {
550
+ id: 'test2',
551
+ description: 'test2a'
552
+ },
553
+ afterReplicaId: test_utils.rid('test2')
554
+ });
555
+ const result1 = await writer.flush();
556
+ const checkpoint1 = result1?.flushed_op ?? 0n;
557
+ // Test batch
558
+ // b
559
+ await writer.save({
560
+ sourceTable,
561
+ tag: storage.SaveOperationTag.INSERT,
562
+ after: {
563
+ id: 'test1',
564
+ description: 'test1b'
565
+ },
566
+ afterReplicaId: test_utils.rid('test1')
567
+ });
568
+ await writer.save({
569
+ sourceTable,
570
+ tag: storage.SaveOperationTag.UPDATE,
571
+ before: {
572
+ id: 'test1'
573
+ },
574
+ beforeReplicaId: test_utils.rid('test1'),
575
+ after: {
576
+ id: 'test2',
577
+ description: 'test2b'
578
+ },
579
+ afterReplicaId: test_utils.rid('test2')
580
+ });
581
+ await writer.save({
582
+ sourceTable,
583
+ tag: storage.SaveOperationTag.UPDATE,
584
+ before: {
585
+ id: 'test2'
586
+ },
587
+ beforeReplicaId: test_utils.rid('test2'),
588
+ after: {
589
+ id: 'test3',
590
+ description: 'test3b'
591
+ },
592
+ afterReplicaId: test_utils.rid('test3')
593
+ });
594
+ // c
595
+ await writer.save({
596
+ sourceTable,
597
+ tag: storage.SaveOperationTag.UPDATE,
598
+ after: {
599
+ id: 'test2',
600
+ description: 'test2c'
601
+ },
602
+ afterReplicaId: test_utils.rid('test2')
603
+ });
604
+ // d
605
+ await writer.save({
606
+ sourceTable,
607
+ tag: storage.SaveOperationTag.INSERT,
608
+ after: {
609
+ id: 'test4',
610
+ description: 'test4d'
611
+ },
612
+ afterReplicaId: test_utils.rid('test4')
613
+ });
614
+ await writer.save({
615
+ sourceTable,
616
+ tag: storage.SaveOperationTag.UPDATE,
617
+ before: {
618
+ id: 'test4'
619
+ },
620
+ beforeReplicaId: test_utils.rid('test4'),
621
+ after: {
622
+ id: 'test5',
623
+ description: 'test5d'
624
+ },
625
+ afterReplicaId: test_utils.rid('test5')
626
+ });
627
+ const result2 = await writer.flush();
628
+ const checkpoint2 = result2.flushed_op;
629
+ const request = bucketRequest(syncRules, 'global[]', checkpoint1);
630
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
631
+ const data = batch[0].chunkData.data.map((d) => {
632
+ return {
633
+ op: d.op,
634
+ object_id: d.object_id,
635
+ data: normalizeOplogData(d.data)
636
+ };
637
+ });
638
+ // Operations must be in this order
639
+ expect(data).toEqual([
732
640
  // b
733
- await writer.save({
734
- sourceTable,
735
- tag: storage.SaveOperationTag.INSERT,
736
- after: {
737
- id: 'test1',
738
- description: 'test1b'
739
- },
740
- afterReplicaId: test_utils.rid('test1')
741
- });
742
- await writer.save({
743
- sourceTable,
744
- tag: storage.SaveOperationTag.UPDATE,
745
- before: {
746
- id: 'test1'
747
- },
748
- beforeReplicaId: test_utils.rid('test1'),
749
- after: {
750
- id: 'test2',
751
- description: 'test2b'
752
- },
753
- afterReplicaId: test_utils.rid('test2')
754
- });
755
- await writer.save({
756
- sourceTable,
757
- tag: storage.SaveOperationTag.UPDATE,
758
- before: {
759
- id: 'test2'
760
- },
761
- beforeReplicaId: test_utils.rid('test2'),
762
- after: {
763
- id: 'test3',
764
- description: 'test3b'
765
- },
766
- afterReplicaId: test_utils.rid('test3')
767
- });
641
+ { op: 'PUT', object_id: 'test1', data: JSON.stringify({ id: 'test1', description: 'test1b' }) },
642
+ { op: 'REMOVE', object_id: 'test1', data: null },
643
+ { op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2b' }) },
644
+ { op: 'REMOVE', object_id: 'test2', data: null },
645
+ { op: 'PUT', object_id: 'test3', data: JSON.stringify({ id: 'test3', description: 'test3b' }) },
768
646
  // c
769
- await writer.save({
770
- sourceTable,
771
- tag: storage.SaveOperationTag.UPDATE,
772
- after: {
773
- id: 'test2',
774
- description: 'test2c'
775
- },
776
- afterReplicaId: test_utils.rid('test2')
777
- });
647
+ { op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2c' }) },
778
648
  // d
779
- await writer.save({
780
- sourceTable,
781
- tag: storage.SaveOperationTag.INSERT,
782
- after: {
783
- id: 'test4',
784
- description: 'test4d'
785
- },
786
- afterReplicaId: test_utils.rid('test4')
787
- });
788
- await writer.save({
789
- sourceTable,
790
- tag: storage.SaveOperationTag.UPDATE,
791
- before: {
792
- id: 'test4'
793
- },
794
- beforeReplicaId: test_utils.rid('test4'),
795
- after: {
796
- id: 'test5',
797
- description: 'test5d'
798
- },
799
- afterReplicaId: test_utils.rid('test5')
800
- });
801
- const result2 = await writer.flush();
802
- const checkpoint2 = result2.flushed_op;
803
- const request = bucketRequest(syncRules, 'global[]', checkpoint1);
804
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
805
- const data = batch[0].chunkData.data.map((d) => {
806
- return {
807
- op: d.op,
808
- object_id: d.object_id,
809
- data: normalizeOplogData(d.data)
810
- };
811
- });
812
- // Operations must be in this order
813
- expect(data).toEqual([
814
- // b
815
- { op: 'PUT', object_id: 'test1', data: JSON.stringify({ id: 'test1', description: 'test1b' }) },
816
- { op: 'REMOVE', object_id: 'test1', data: null },
817
- { op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2b' }) },
818
- { op: 'REMOVE', object_id: 'test2', data: null },
819
- { op: 'PUT', object_id: 'test3', data: JSON.stringify({ id: 'test3', description: 'test3b' }) },
820
- // c
821
- { op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2c' }) },
822
- // d
823
- { op: 'PUT', object_id: 'test4', data: JSON.stringify({ id: 'test4', description: 'test4d' }) },
824
- { op: 'REMOVE', object_id: 'test4', data: null },
825
- { op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
826
- ]);
827
- }
828
- catch (e_11) {
829
- env_11.error = e_11;
830
- env_11.hasError = true;
831
- }
832
- finally {
833
- const result_11 = __disposeResources(env_11);
834
- if (result_11)
835
- await result_11;
836
- }
649
+ { op: 'PUT', object_id: 'test4', data: JSON.stringify({ id: 'test4', description: 'test4d' }) },
650
+ { op: 'REMOVE', object_id: 'test4', data: null },
651
+ { op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
652
+ ]);
837
653
  });
838
654
  test('changed data with replica identity full', async () => {
839
- const env_12 = { stack: [], error: void 0, hasError: false };
840
- try {
841
- function rid2(id, description) {
842
- return getUuidReplicaIdentityBson({ id, description }, [
843
- { name: 'id', type: 'VARCHAR', typeId: 25 },
844
- { name: 'description', type: 'VARCHAR', typeId: 25 }
845
- ]);
846
- }
847
- const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
848
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
849
- bucket_definitions:
850
- global:
851
- data:
852
- - SELECT id, description FROM "test"
853
- `, {
854
- storageVersion
855
- }));
856
- const bucketStorage = factory.getInstance(syncRules);
857
- const writer = __addDisposableResource(env_12, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
858
- const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
859
- // Pre-setup
860
- await writer.markAllSnapshotDone('1/1');
861
- await writer.save({
862
- sourceTable,
863
- tag: storage.SaveOperationTag.INSERT,
864
- after: {
865
- id: 'test1',
866
- description: 'test1a'
867
- },
868
- afterReplicaId: rid2('test1', 'test1a')
869
- });
870
- const result1 = await writer.flush();
871
- const checkpoint1 = result1?.flushed_op ?? 0n;
872
- // Unchanged, but has a before id
873
- await writer.save({
874
- sourceTable,
875
- tag: storage.SaveOperationTag.UPDATE,
876
- before: {
877
- id: 'test1',
878
- description: 'test1a'
879
- },
880
- beforeReplicaId: rid2('test1', 'test1a'),
881
- after: {
882
- id: 'test1',
883
- description: 'test1b'
884
- },
885
- afterReplicaId: rid2('test1', 'test1b')
886
- });
887
- const result2 = await writer.flush();
888
- // Delete
889
- await writer.save({
890
- sourceTable,
891
- tag: storage.SaveOperationTag.DELETE,
892
- before: {
893
- id: 'test1',
894
- description: 'test1b'
895
- },
896
- beforeReplicaId: rid2('test1', 'test1b'),
897
- after: undefined
898
- });
899
- const result3 = await writer.flush();
900
- const checkpoint3 = result3.flushed_op;
901
- const request = bucketRequest(syncRules, 'global[]');
902
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
903
- const data = batch[0].chunkData.data.map((d) => {
904
- return {
905
- op: d.op,
906
- object_id: d.object_id,
907
- data: normalizeOplogData(d.data),
908
- subkey: d.subkey
909
- };
910
- });
911
- // Operations must be in this order
912
- expect(data).toEqual([
913
- // 2
914
- // The REMOVE is expected because the subkey changes
915
- {
916
- op: 'REMOVE',
917
- object_id: 'test1',
918
- data: null,
919
- subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
920
- },
921
- {
922
- op: 'PUT',
923
- object_id: 'test1',
924
- data: JSON.stringify({ id: 'test1', description: 'test1b' }),
925
- subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
926
- },
927
- // 3
928
- {
929
- op: 'REMOVE',
930
- object_id: 'test1',
931
- data: null,
932
- subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
933
- }
934
- ]);
935
- }
936
- catch (e_12) {
937
- env_12.error = e_12;
938
- env_12.hasError = true;
939
- }
940
- finally {
941
- const result_12 = __disposeResources(env_12);
942
- if (result_12)
943
- await result_12;
655
+ function rid2(id, description) {
656
+ return getUuidReplicaIdentityBson({ id, description }, [
657
+ { name: 'id', type: 'VARCHAR', typeId: 25 },
658
+ { name: 'description', type: 'VARCHAR', typeId: 25 }
659
+ ]);
944
660
  }
945
- });
946
- test('unchanged data with replica identity full', async () => {
947
- const env_13 = { stack: [], error: void 0, hasError: false };
948
- try {
949
- function rid2(id, description) {
950
- return getUuidReplicaIdentityBson({ id, description }, [
951
- { name: 'id', type: 'VARCHAR', typeId: 25 },
952
- { name: 'description', type: 'VARCHAR', typeId: 25 }
953
- ]);
954
- }
955
- const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
956
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
661
+ await using factory = await generateStorageFactory();
662
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
957
663
  bucket_definitions:
958
664
  global:
959
665
  data:
960
666
  - SELECT id, description FROM "test"
961
667
  `, {
962
- storageVersion
963
- }));
964
- const bucketStorage = factory.getInstance(syncRules);
965
- const writer = __addDisposableResource(env_13, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
966
- const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
967
- // Pre-setup
968
- await writer.markAllSnapshotDone('1/1');
969
- await writer.save({
970
- sourceTable,
971
- tag: storage.SaveOperationTag.INSERT,
972
- after: {
973
- id: 'test1',
974
- description: 'test1a'
975
- },
976
- afterReplicaId: rid2('test1', 'test1a')
977
- });
978
- const result1 = await writer.flush();
979
- const checkpoint1 = result1?.flushed_op ?? 0n;
980
- // Unchanged, but has a before id
981
- await writer.save({
982
- sourceTable,
983
- tag: storage.SaveOperationTag.UPDATE,
984
- before: {
985
- id: 'test1',
986
- description: 'test1a'
987
- },
988
- beforeReplicaId: rid2('test1', 'test1a'),
989
- after: {
990
- id: 'test1',
991
- description: 'test1a'
992
- },
993
- afterReplicaId: rid2('test1', 'test1a')
994
- });
995
- const result2 = await writer.flush();
996
- // Delete
997
- await writer.save({
998
- sourceTable,
999
- tag: storage.SaveOperationTag.DELETE,
1000
- before: {
1001
- id: 'test1',
1002
- description: 'test1a'
1003
- },
1004
- beforeReplicaId: rid2('test1', 'test1a'),
1005
- after: undefined
1006
- });
1007
- const result3 = await writer.flush();
1008
- const checkpoint3 = result3.flushed_op;
1009
- const request = bucketRequest(syncRules, 'global[]');
1010
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
1011
- const data = batch[0].chunkData.data.map((d) => {
1012
- return {
1013
- op: d.op,
1014
- object_id: d.object_id,
1015
- data: normalizeOplogData(d.data),
1016
- subkey: d.subkey
1017
- };
1018
- });
1019
- // Operations must be in this order
1020
- expect(data).toEqual([
1021
- // 2
1022
- {
1023
- op: 'PUT',
1024
- object_id: 'test1',
1025
- data: JSON.stringify({ id: 'test1', description: 'test1a' }),
1026
- subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
1027
- },
1028
- // 3
1029
- {
1030
- op: 'REMOVE',
1031
- object_id: 'test1',
1032
- data: null,
1033
- subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
1034
- }
668
+ storageVersion
669
+ }));
670
+ const bucketStorage = factory.getInstance(syncRules);
671
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
672
+ const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
673
+ // Pre-setup
674
+ await writer.markAllSnapshotDone('1/1');
675
+ await writer.save({
676
+ sourceTable,
677
+ tag: storage.SaveOperationTag.INSERT,
678
+ after: {
679
+ id: 'test1',
680
+ description: 'test1a'
681
+ },
682
+ afterReplicaId: rid2('test1', 'test1a')
683
+ });
684
+ const result1 = await writer.flush();
685
+ const checkpoint1 = result1?.flushed_op ?? 0n;
686
+ // Unchanged, but has a before id
687
+ await writer.save({
688
+ sourceTable,
689
+ tag: storage.SaveOperationTag.UPDATE,
690
+ before: {
691
+ id: 'test1',
692
+ description: 'test1a'
693
+ },
694
+ beforeReplicaId: rid2('test1', 'test1a'),
695
+ after: {
696
+ id: 'test1',
697
+ description: 'test1b'
698
+ },
699
+ afterReplicaId: rid2('test1', 'test1b')
700
+ });
701
+ const result2 = await writer.flush();
702
+ // Delete
703
+ await writer.save({
704
+ sourceTable,
705
+ tag: storage.SaveOperationTag.DELETE,
706
+ before: {
707
+ id: 'test1',
708
+ description: 'test1b'
709
+ },
710
+ beforeReplicaId: rid2('test1', 'test1b'),
711
+ after: undefined
712
+ });
713
+ const result3 = await writer.flush();
714
+ const checkpoint3 = result3.flushed_op;
715
+ const request = bucketRequest(syncRules, 'global[]');
716
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
717
+ const data = batch[0].chunkData.data.map((d) => {
718
+ return {
719
+ op: d.op,
720
+ object_id: d.object_id,
721
+ data: normalizeOplogData(d.data),
722
+ subkey: d.subkey
723
+ };
724
+ });
725
+ // Operations must be in this order
726
+ expect(data).toEqual([
727
+ // 2
728
+ // The REMOVE is expected because the subkey changes
729
+ {
730
+ op: 'REMOVE',
731
+ object_id: 'test1',
732
+ data: null,
733
+ subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
734
+ },
735
+ {
736
+ op: 'PUT',
737
+ object_id: 'test1',
738
+ data: JSON.stringify({ id: 'test1', description: 'test1b' }),
739
+ subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
740
+ },
741
+ // 3
742
+ {
743
+ op: 'REMOVE',
744
+ object_id: 'test1',
745
+ data: null,
746
+ subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
747
+ }
748
+ ]);
749
+ });
750
+ test('unchanged data with replica identity full', async () => {
751
+ function rid2(id, description) {
752
+ return getUuidReplicaIdentityBson({ id, description }, [
753
+ { name: 'id', type: 'VARCHAR', typeId: 25 },
754
+ { name: 'description', type: 'VARCHAR', typeId: 25 }
1035
755
  ]);
1036
756
  }
1037
- catch (e_13) {
1038
- env_13.error = e_13;
1039
- env_13.hasError = true;
1040
- }
1041
- finally {
1042
- const result_13 = __disposeResources(env_13);
1043
- if (result_13)
1044
- await result_13;
1045
- }
757
+ await using factory = await generateStorageFactory();
758
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
759
+ bucket_definitions:
760
+ global:
761
+ data:
762
+ - SELECT id, description FROM "test"
763
+ `, {
764
+ storageVersion
765
+ }));
766
+ const bucketStorage = factory.getInstance(syncRules);
767
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
768
+ const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
769
+ // Pre-setup
770
+ await writer.markAllSnapshotDone('1/1');
771
+ await writer.save({
772
+ sourceTable,
773
+ tag: storage.SaveOperationTag.INSERT,
774
+ after: {
775
+ id: 'test1',
776
+ description: 'test1a'
777
+ },
778
+ afterReplicaId: rid2('test1', 'test1a')
779
+ });
780
+ const result1 = await writer.flush();
781
+ const checkpoint1 = result1?.flushed_op ?? 0n;
782
+ // Unchanged, but has a before id
783
+ await writer.save({
784
+ sourceTable,
785
+ tag: storage.SaveOperationTag.UPDATE,
786
+ before: {
787
+ id: 'test1',
788
+ description: 'test1a'
789
+ },
790
+ beforeReplicaId: rid2('test1', 'test1a'),
791
+ after: {
792
+ id: 'test1',
793
+ description: 'test1a'
794
+ },
795
+ afterReplicaId: rid2('test1', 'test1a')
796
+ });
797
+ const result2 = await writer.flush();
798
+ // Delete
799
+ await writer.save({
800
+ sourceTable,
801
+ tag: storage.SaveOperationTag.DELETE,
802
+ before: {
803
+ id: 'test1',
804
+ description: 'test1a'
805
+ },
806
+ beforeReplicaId: rid2('test1', 'test1a'),
807
+ after: undefined
808
+ });
809
+ const result3 = await writer.flush();
810
+ const checkpoint3 = result3.flushed_op;
811
+ const request = bucketRequest(syncRules, 'global[]');
812
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
813
+ const data = batch[0].chunkData.data.map((d) => {
814
+ return {
815
+ op: d.op,
816
+ object_id: d.object_id,
817
+ data: normalizeOplogData(d.data),
818
+ subkey: d.subkey
819
+ };
820
+ });
821
+ // Operations must be in this order
822
+ expect(data).toEqual([
823
+ // 2
824
+ {
825
+ op: 'PUT',
826
+ object_id: 'test1',
827
+ data: JSON.stringify({ id: 'test1', description: 'test1a' }),
828
+ subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
829
+ },
830
+ // 3
831
+ {
832
+ op: 'REMOVE',
833
+ object_id: 'test1',
834
+ data: null,
835
+ subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
836
+ }
837
+ ]);
1046
838
  });
1047
839
  test('large batch', async () => {
1048
- const env_14 = { stack: [], error: void 0, hasError: false };
1049
- try {
1050
- // Test syncing a batch of data that is small in count,
1051
- // but large enough in size to be split over multiple returned batches.
1052
- // The specific batch splits is an implementation detail of the storage driver,
1053
- // and the test will have to updated when other implementations are added.
1054
- const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
1055
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
840
+ // Test syncing a batch of data that is small in count,
841
+ // but large enough in size to be split over multiple returned batches.
842
+ // The specific batch splits is an implementation detail of the storage driver,
843
+ // and the test will have to updated when other implementations are added.
844
+ await using factory = await generateStorageFactory();
845
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1056
846
  bucket_definitions:
1057
847
  global:
1058
848
  data:
1059
849
  - SELECT id, description FROM "%"
1060
850
  `, {
1061
- storageVersion
1062
- }));
1063
- const bucketStorage = factory.getInstance(syncRules);
1064
- const writer = __addDisposableResource(env_14, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1065
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1066
- await writer.markAllSnapshotDone('1/1');
1067
- const largeDescription = '0123456789'.repeat(12_000_00);
1068
- await writer.save({
1069
- sourceTable,
1070
- tag: storage.SaveOperationTag.INSERT,
1071
- after: {
1072
- id: 'test1',
1073
- description: 'test1'
1074
- },
1075
- afterReplicaId: test_utils.rid('test1')
1076
- });
1077
- await writer.save({
1078
- sourceTable,
1079
- tag: storage.SaveOperationTag.INSERT,
1080
- after: {
1081
- id: 'large1',
1082
- description: largeDescription
1083
- },
1084
- afterReplicaId: test_utils.rid('large1')
1085
- });
1086
- // Large enough to split the returned batch
1087
- await writer.save({
1088
- sourceTable,
1089
- tag: storage.SaveOperationTag.INSERT,
1090
- after: {
1091
- id: 'large2',
1092
- description: largeDescription
1093
- },
1094
- afterReplicaId: test_utils.rid('large2')
1095
- });
1096
- await writer.save({
1097
- sourceTable,
1098
- tag: storage.SaveOperationTag.INSERT,
1099
- after: {
1100
- id: 'test3',
1101
- description: 'test3'
1102
- },
1103
- afterReplicaId: test_utils.rid('test3')
1104
- });
1105
- await writer.commit('1/1');
1106
- const { checkpoint } = await bucketStorage.getCheckpoint();
1107
- const options = {
1108
- chunkLimitBytes: 16 * 1024 * 1024
1109
- };
1110
- const request = bucketRequest(syncRules, 'global[]');
1111
- const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
1112
- expect(test_utils.getBatchData(batch1)).toEqual([
1113
- { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1114
- { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
1115
- ]);
1116
- expect(test_utils.getBatchMeta(batch1)).toEqual({
1117
- after: '0',
1118
- has_more: true,
1119
- next_after: '2'
1120
- });
1121
- const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
1122
- expect(test_utils.getBatchData(batch2)).toEqual([
1123
- { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
1124
- { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
1125
- ]);
1126
- expect(test_utils.getBatchMeta(batch2)).toEqual({
1127
- after: '2',
1128
- has_more: false,
1129
- next_after: '4'
1130
- });
1131
- const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
1132
- expect(test_utils.getBatchData(batch3)).toEqual([]);
1133
- expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1134
- }
1135
- catch (e_14) {
1136
- env_14.error = e_14;
1137
- env_14.hasError = true;
1138
- }
1139
- finally {
1140
- const result_14 = __disposeResources(env_14);
1141
- if (result_14)
1142
- await result_14;
1143
- }
851
+ storageVersion
852
+ }));
853
+ const bucketStorage = factory.getInstance(syncRules);
854
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
855
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
856
+ await writer.markAllSnapshotDone('1/1');
857
+ const largeDescription = '0123456789'.repeat(12_000_00);
858
+ await writer.save({
859
+ sourceTable,
860
+ tag: storage.SaveOperationTag.INSERT,
861
+ after: {
862
+ id: 'test1',
863
+ description: 'test1'
864
+ },
865
+ afterReplicaId: test_utils.rid('test1')
866
+ });
867
+ await writer.save({
868
+ sourceTable,
869
+ tag: storage.SaveOperationTag.INSERT,
870
+ after: {
871
+ id: 'large1',
872
+ description: largeDescription
873
+ },
874
+ afterReplicaId: test_utils.rid('large1')
875
+ });
876
+ // Large enough to split the returned batch
877
+ await writer.save({
878
+ sourceTable,
879
+ tag: storage.SaveOperationTag.INSERT,
880
+ after: {
881
+ id: 'large2',
882
+ description: largeDescription
883
+ },
884
+ afterReplicaId: test_utils.rid('large2')
885
+ });
886
+ await writer.save({
887
+ sourceTable,
888
+ tag: storage.SaveOperationTag.INSERT,
889
+ after: {
890
+ id: 'test3',
891
+ description: 'test3'
892
+ },
893
+ afterReplicaId: test_utils.rid('test3')
894
+ });
895
+ await writer.commit('1/1');
896
+ const { checkpoint } = await bucketStorage.getCheckpoint();
897
+ const options = {
898
+ chunkLimitBytes: 16 * 1024 * 1024
899
+ };
900
+ const request = bucketRequest(syncRules, 'global[]');
901
+ const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
902
+ expect(test_utils.getBatchData(batch1)).toEqual([
903
+ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
904
+ { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
905
+ ]);
906
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
907
+ after: '0',
908
+ has_more: true,
909
+ next_after: '2'
910
+ });
911
+ const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
912
+ expect(test_utils.getBatchData(batch2)).toEqual([
913
+ { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
914
+ { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
915
+ ]);
916
+ expect(test_utils.getBatchMeta(batch2)).toEqual({
917
+ after: '2',
918
+ has_more: false,
919
+ next_after: '4'
920
+ });
921
+ const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
922
+ expect(test_utils.getBatchData(batch3)).toEqual([]);
923
+ expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1144
924
  });
1145
925
  test('long batch', async () => {
1146
- const env_15 = { stack: [], error: void 0, hasError: false };
1147
- try {
1148
- // Test syncing a batch of data that is limited by count.
1149
- const factory = __addDisposableResource(env_15, await generateStorageFactory(), true);
1150
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
926
+ // Test syncing a batch of data that is limited by count.
927
+ await using factory = await generateStorageFactory();
928
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1151
929
  bucket_definitions:
1152
930
  global:
1153
931
  data:
1154
932
  - SELECT id, description FROM "%"
1155
933
  `, {
1156
- storageVersion
1157
- }));
934
+ storageVersion
935
+ }));
936
+ const bucketStorage = factory.getInstance(syncRules);
937
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
938
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
939
+ await writer.markAllSnapshotDone('1/1');
940
+ for (let i = 1; i <= 6; i++) {
941
+ await writer.save({
942
+ sourceTable,
943
+ tag: storage.SaveOperationTag.INSERT,
944
+ after: {
945
+ id: `test${i}`,
946
+ description: `test${i}`
947
+ },
948
+ afterReplicaId: `test${i}`
949
+ });
950
+ }
951
+ await writer.commit('1/1');
952
+ const { checkpoint } = await bucketStorage.getCheckpoint();
953
+ const request = bucketRequest(syncRules, 'global[]');
954
+ const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
955
+ expect(test_utils.getBatchData(batch1)).toEqual([
956
+ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
957
+ { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
958
+ { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
959
+ { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
960
+ ]);
961
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
962
+ after: '0',
963
+ has_more: true,
964
+ next_after: '4'
965
+ });
966
+ const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
967
+ limit: 4
968
+ }));
969
+ expect(test_utils.getBatchData(batch2)).toEqual([
970
+ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
971
+ { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
972
+ ]);
973
+ expect(test_utils.getBatchMeta(batch2)).toEqual({
974
+ after: '4',
975
+ has_more: false,
976
+ next_after: '6'
977
+ });
978
+ const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
979
+ limit: 4
980
+ }));
981
+ expect(test_utils.getBatchData(batch3)).toEqual([]);
982
+ expect(test_utils.getBatchMeta(batch3)).toEqual(null);
983
+ });
984
+ describe('batch has_more', () => {
985
+ const setup = async (options) => {
986
+ await using factory = await generateStorageFactory();
987
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
988
+ bucket_definitions:
989
+ global1:
990
+ data:
991
+ - SELECT id, description FROM test WHERE bucket = 'global1'
992
+ global2:
993
+ data:
994
+ - SELECT id, description FROM test WHERE bucket = 'global2'
995
+ `, { storageVersion }));
1158
996
  const bucketStorage = factory.getInstance(syncRules);
1159
- const writer = __addDisposableResource(env_15, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
997
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1160
998
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1161
999
  await writer.markAllSnapshotDone('1/1');
1162
- for (let i = 1; i <= 6; i++) {
1000
+ for (let i = 1; i <= 10; i++) {
1163
1001
  await writer.save({
1164
1002
  sourceTable,
1165
1003
  tag: storage.SaveOperationTag.INSERT,
1166
1004
  after: {
1167
1005
  id: `test${i}`,
1168
- description: `test${i}`
1006
+ description: `test${i}`,
1007
+ bucket: i == 1 ? 'global1' : 'global2'
1169
1008
  },
1170
1009
  afterReplicaId: `test${i}`
1171
1010
  });
1172
1011
  }
1173
1012
  await writer.commit('1/1');
1174
1013
  const { checkpoint } = await bucketStorage.getCheckpoint();
1175
- const request = bucketRequest(syncRules, 'global[]');
1176
- const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
1177
- expect(test_utils.getBatchData(batch1)).toEqual([
1178
- { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
1179
- { op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
1180
- { op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
1181
- { op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
1182
- ]);
1183
- expect(test_utils.getBatchMeta(batch1)).toEqual({
1184
- after: '0',
1185
- has_more: true,
1186
- next_after: '4'
1187
- });
1188
- const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
1189
- limit: 4
1190
- }));
1191
- expect(test_utils.getBatchData(batch2)).toEqual([
1192
- { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
1193
- { op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
1194
- ]);
1195
- expect(test_utils.getBatchMeta(batch2)).toEqual({
1196
- after: '4',
1197
- has_more: false,
1198
- next_after: '6'
1199
- });
1200
- const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
1201
- limit: 4
1202
- }));
1203
- expect(test_utils.getBatchData(batch3)).toEqual([]);
1204
- expect(test_utils.getBatchMeta(batch3)).toEqual(null);
1205
- }
1206
- catch (e_15) {
1207
- env_15.error = e_15;
1208
- env_15.hasError = true;
1209
- }
1210
- finally {
1211
- const result_15 = __disposeResources(env_15);
1212
- if (result_15)
1213
- await result_15;
1214
- }
1215
- });
1216
- describe('batch has_more', () => {
1217
- const setup = async (options) => {
1218
- const env_16 = { stack: [], error: void 0, hasError: false };
1219
- try {
1220
- const factory = __addDisposableResource(env_16, await generateStorageFactory(), true);
1221
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1222
- bucket_definitions:
1223
- global1:
1224
- data:
1225
- - SELECT id, description FROM test WHERE bucket = 'global1'
1226
- global2:
1227
- data:
1228
- - SELECT id, description FROM test WHERE bucket = 'global2'
1229
- `, { storageVersion }));
1230
- const bucketStorage = factory.getInstance(syncRules);
1231
- const writer = __addDisposableResource(env_16, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1232
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1233
- await writer.markAllSnapshotDone('1/1');
1234
- for (let i = 1; i <= 10; i++) {
1235
- await writer.save({
1236
- sourceTable,
1237
- tag: storage.SaveOperationTag.INSERT,
1238
- after: {
1239
- id: `test${i}`,
1240
- description: `test${i}`,
1241
- bucket: i == 1 ? 'global1' : 'global2'
1242
- },
1243
- afterReplicaId: `test${i}`
1244
- });
1245
- }
1246
- await writer.commit('1/1');
1247
- const { checkpoint } = await bucketStorage.getCheckpoint();
1248
- const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
1249
- const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
1250
- const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
1251
- return { batch, global1Request, global2Request };
1252
- }
1253
- catch (e_16) {
1254
- env_16.error = e_16;
1255
- env_16.hasError = true;
1256
- }
1257
- finally {
1258
- const result_16 = __disposeResources(env_16);
1259
- if (result_16)
1260
- await result_16;
1261
- }
1014
+ const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
1015
+ const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
1016
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
1017
+ return { batch, global1Request, global2Request };
1262
1018
  };
1263
1019
  test('batch has_more (1)', async () => {
1264
1020
  const { batch, global1Request, global2Request } = await setup({ limit: 5 });
@@ -1349,330 +1105,247 @@ bucket_definitions:
1349
1105
  });
1350
1106
  });
1351
1107
  test('empty storage metrics', async () => {
1352
- const env_17 = { stack: [], error: void 0, hasError: false };
1353
- try {
1354
- const f = __addDisposableResource(env_17, await generateStorageFactory({ dropAll: true }), true);
1355
- const metrics = await f.getStorageMetrics();
1356
- expect(metrics).toEqual({
1357
- operations_size_bytes: 0,
1358
- parameters_size_bytes: 0,
1359
- replication_size_bytes: 0
1360
- });
1361
- const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
1362
- const storage = f.getInstance(r.persisted_sync_rules);
1363
- const writer = __addDisposableResource(env_17, await storage.createWriter(test_utils.BATCH_OPTIONS), true);
1364
- await writer.markAllSnapshotDone('1/0');
1365
- await writer.keepalive('1/0');
1366
- await f.getStorageMetrics();
1367
- }
1368
- catch (e_17) {
1369
- env_17.error = e_17;
1370
- env_17.hasError = true;
1371
- }
1372
- finally {
1373
- const result_17 = __disposeResources(env_17);
1374
- if (result_17)
1375
- await result_17;
1376
- }
1108
+ await using f = await generateStorageFactory({ dropAll: true });
1109
+ const metrics = await f.getStorageMetrics();
1110
+ expect(metrics).toEqual({
1111
+ operations_size_bytes: 0,
1112
+ parameters_size_bytes: 0,
1113
+ replication_size_bytes: 0
1114
+ });
1115
+ const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
1116
+ const storage = f.getInstance(r.persisted_sync_rules);
1117
+ await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
1118
+ await writer.markAllSnapshotDone('1/0');
1119
+ await writer.keepalive('1/0');
1120
+ await f.getStorageMetrics();
1121
+ // We don't care about the specific values here
1377
1122
  });
1378
1123
  test('op_id initialization edge case', async () => {
1379
- const env_18 = { stack: [], error: void 0, hasError: false };
1380
- try {
1381
- // Test syncing a batch of data that is small in count,
1382
- // but large enough in size to be split over multiple returned chunks.
1383
- // Similar to the above test, but splits over 1MB chunks.
1384
- const factory = __addDisposableResource(env_18, await generateStorageFactory(), true);
1385
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1124
+ // Test syncing a batch of data that is small in count,
1125
+ // but large enough in size to be split over multiple returned chunks.
1126
+ // Similar to the above test, but splits over 1MB chunks.
1127
+ await using factory = await generateStorageFactory();
1128
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1386
1129
  bucket_definitions:
1387
1130
  global:
1388
1131
  data:
1389
1132
  - SELECT id FROM test
1390
1133
  - SELECT id FROM test_ignore WHERE false
1391
1134
  `, {
1392
- storageVersion
1393
- }));
1394
- const bucketStorage = factory.getInstance(syncRules);
1395
- const writer = __addDisposableResource(env_18, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1396
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
1397
- const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
1398
- await writer.markAllSnapshotDone('1/1');
1399
- // This saves a record to current_data, but not bucket_data.
1400
- // This causes a checkpoint to be created without increasing the op_id sequence.
1401
- await writer.save({
1402
- sourceTable: sourceTableIgnore,
1403
- tag: storage.SaveOperationTag.INSERT,
1404
- after: {
1405
- id: 'test1'
1406
- },
1407
- afterReplicaId: test_utils.rid('test1')
1408
- });
1409
- const result1 = await writer.flush();
1410
- const checkpoint1 = result1.flushed_op;
1411
- await writer.save({
1412
- sourceTable: sourceTable,
1413
- tag: storage.SaveOperationTag.INSERT,
1414
- after: {
1415
- id: 'test2'
1416
- },
1417
- afterReplicaId: test_utils.rid('test2')
1418
- });
1419
- const result2 = await writer.flush();
1420
- const checkpoint2 = result2.flushed_op;
1421
- // we expect 0n and 1n, or 1n and 2n.
1422
- expect(checkpoint2).toBeGreaterThan(checkpoint1);
1423
- }
1424
- catch (e_18) {
1425
- env_18.error = e_18;
1426
- env_18.hasError = true;
1427
- }
1428
- finally {
1429
- const result_18 = __disposeResources(env_18);
1430
- if (result_18)
1431
- await result_18;
1432
- }
1135
+ storageVersion
1136
+ }));
1137
+ const bucketStorage = factory.getInstance(syncRules);
1138
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1139
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
1140
+ const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
1141
+ await writer.markAllSnapshotDone('1/1');
1142
+ // This saves a record to current_data, but not bucket_data.
1143
+ // This causes a checkpoint to be created without increasing the op_id sequence.
1144
+ await writer.save({
1145
+ sourceTable: sourceTableIgnore,
1146
+ tag: storage.SaveOperationTag.INSERT,
1147
+ after: {
1148
+ id: 'test1'
1149
+ },
1150
+ afterReplicaId: test_utils.rid('test1')
1151
+ });
1152
+ const result1 = await writer.flush();
1153
+ const checkpoint1 = result1.flushed_op;
1154
+ await writer.save({
1155
+ sourceTable: sourceTable,
1156
+ tag: storage.SaveOperationTag.INSERT,
1157
+ after: {
1158
+ id: 'test2'
1159
+ },
1160
+ afterReplicaId: test_utils.rid('test2')
1161
+ });
1162
+ const result2 = await writer.flush();
1163
+ const checkpoint2 = result2.flushed_op;
1164
+ // we expect 0n and 1n, or 1n and 2n.
1165
+ expect(checkpoint2).toBeGreaterThan(checkpoint1);
1433
1166
  });
1434
1167
  test('unchanged checksums', async () => {
1435
- const env_19 = { stack: [], error: void 0, hasError: false };
1436
- try {
1437
- const factory = __addDisposableResource(env_19, await generateStorageFactory(), true);
1438
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1168
+ await using factory = await generateStorageFactory();
1169
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1439
1170
  bucket_definitions:
1440
1171
  global:
1441
1172
  data:
1442
1173
  - SELECT client_id as id, description FROM "%"
1443
1174
  `, {
1444
- storageVersion
1445
- }));
1446
- const bucketStorage = factory.getInstance(syncRules);
1447
- const writer = __addDisposableResource(env_19, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1448
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1449
- await writer.markAllSnapshotDone('1/1');
1450
- await writer.save({
1451
- sourceTable,
1452
- tag: storage.SaveOperationTag.INSERT,
1453
- after: {
1454
- id: 'test1',
1455
- description: 'test1a'
1456
- },
1457
- afterReplicaId: test_utils.rid('test1')
1458
- });
1459
- await writer.commit('1/1');
1460
- const { checkpoint } = await bucketStorage.getCheckpoint();
1461
- const request = bucketRequest(syncRules, 'global[]');
1462
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
1463
- expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1464
- const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
1465
- expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1466
- }
1467
- catch (e_19) {
1468
- env_19.error = e_19;
1469
- env_19.hasError = true;
1470
- }
1471
- finally {
1472
- const result_19 = __disposeResources(env_19);
1473
- if (result_19)
1474
- await result_19;
1475
- }
1175
+ storageVersion
1176
+ }));
1177
+ const bucketStorage = factory.getInstance(syncRules);
1178
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1179
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1180
+ await writer.markAllSnapshotDone('1/1');
1181
+ await writer.save({
1182
+ sourceTable,
1183
+ tag: storage.SaveOperationTag.INSERT,
1184
+ after: {
1185
+ id: 'test1',
1186
+ description: 'test1a'
1187
+ },
1188
+ afterReplicaId: test_utils.rid('test1')
1189
+ });
1190
+ await writer.commit('1/1');
1191
+ const { checkpoint } = await bucketStorage.getCheckpoint();
1192
+ const request = bucketRequest(syncRules, 'global[]');
1193
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
1194
+ expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1195
+ const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
1196
+ expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
1476
1197
  });
1477
1198
  testChecksumBatching(config);
1478
1199
  test('empty checkpoints (1)', async () => {
1479
- const env_20 = { stack: [], error: void 0, hasError: false };
1480
- try {
1481
- const factory = __addDisposableResource(env_20, await generateStorageFactory(), true);
1482
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1200
+ await using factory = await generateStorageFactory();
1201
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1483
1202
  bucket_definitions:
1484
1203
  global:
1485
1204
  data:
1486
1205
  - SELECT id, description FROM "%"
1487
1206
  `, { storageVersion }));
1488
- const bucketStorage = factory.getInstance(syncRules);
1489
- const writer = __addDisposableResource(env_20, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1490
- await writer.markAllSnapshotDone('1/1');
1491
- await writer.commit('1/1');
1492
- const cp1 = await bucketStorage.getCheckpoint();
1493
- expect(cp1.lsn).toEqual('1/1');
1494
- await writer.commit('2/1', { createEmptyCheckpoints: true });
1495
- const cp2 = await bucketStorage.getCheckpoint();
1496
- expect(cp2.lsn).toEqual('2/1');
1497
- await writer.keepalive('3/1');
1498
- const cp3 = await bucketStorage.getCheckpoint();
1499
- expect(cp3.lsn).toEqual('3/1');
1500
- // For the last one, we skip creating empty checkpoints
1501
- // This means the LSN stays at 3/1.
1502
- await writer.commit('4/1', { createEmptyCheckpoints: false });
1503
- const cp4 = await bucketStorage.getCheckpoint();
1504
- expect(cp4.lsn).toEqual('3/1');
1505
- }
1506
- catch (e_20) {
1507
- env_20.error = e_20;
1508
- env_20.hasError = true;
1509
- }
1510
- finally {
1511
- const result_20 = __disposeResources(env_20);
1512
- if (result_20)
1513
- await result_20;
1514
- }
1207
+ const bucketStorage = factory.getInstance(syncRules);
1208
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1209
+ await writer.markAllSnapshotDone('1/1');
1210
+ await writer.commit('1/1');
1211
+ const cp1 = await bucketStorage.getCheckpoint();
1212
+ expect(cp1.lsn).toEqual('1/1');
1213
+ await writer.commit('2/1', { createEmptyCheckpoints: true });
1214
+ const cp2 = await bucketStorage.getCheckpoint();
1215
+ expect(cp2.lsn).toEqual('2/1');
1216
+ await writer.keepalive('3/1');
1217
+ const cp3 = await bucketStorage.getCheckpoint();
1218
+ expect(cp3.lsn).toEqual('3/1');
1219
+ // For the last one, we skip creating empty checkpoints
1220
+ // This means the LSN stays at 3/1.
1221
+ await writer.commit('4/1', { createEmptyCheckpoints: false });
1222
+ const cp4 = await bucketStorage.getCheckpoint();
1223
+ expect(cp4.lsn).toEqual('3/1');
1515
1224
  });
1516
1225
  test('empty checkpoints (2)', async () => {
1517
- const env_21 = { stack: [], error: void 0, hasError: false };
1518
- try {
1519
- const factory = __addDisposableResource(env_21, await generateStorageFactory(), true);
1520
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1226
+ await using factory = await generateStorageFactory();
1227
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1521
1228
  bucket_definitions:
1522
1229
  global:
1523
1230
  data:
1524
1231
  - SELECT id, description FROM "%"
1525
1232
  `, {
1526
- storageVersion
1527
- }));
1528
- const bucketStorage = factory.getInstance(syncRules);
1529
- const writer1 = __addDisposableResource(env_21, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1530
- const writer2 = __addDisposableResource(env_21, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1531
- const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
1532
- // We simulate two concurrent batches, but sequential calls are enough for this test.
1533
- await writer1.markAllSnapshotDone('1/1');
1534
- await writer1.commit('1/1');
1535
- await writer1.commit('2/1', { createEmptyCheckpoints: false });
1536
- const cp2 = await bucketStorage.getCheckpoint();
1537
- expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
1538
- await writer2.save({
1539
- sourceTable,
1540
- tag: storage.SaveOperationTag.INSERT,
1541
- after: {
1542
- id: 'test1',
1543
- description: 'test1a'
1544
- },
1545
- afterReplicaId: test_utils.rid('test1')
1546
- });
1547
- // This simulates what happens on a snapshot processor.
1548
- // This may later change to a flush() rather than commit().
1549
- await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
1550
- const cp3 = await bucketStorage.getCheckpoint();
1551
- expect(cp3.lsn).toEqual('1/1'); // Still unchanged
1552
- // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
1553
- await writer1.commit('4/1', { createEmptyCheckpoints: false });
1554
- const cp4 = await bucketStorage.getCheckpoint();
1555
- expect(cp4.lsn).toEqual('4/1');
1556
- }
1557
- catch (e_21) {
1558
- env_21.error = e_21;
1559
- env_21.hasError = true;
1560
- }
1561
- finally {
1562
- const result_21 = __disposeResources(env_21);
1563
- if (result_21)
1564
- await result_21;
1565
- }
1233
+ storageVersion
1234
+ }));
1235
+ const bucketStorage = factory.getInstance(syncRules);
1236
+ await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1237
+ await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1238
+ const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
1239
+ // We simulate two concurrent batches, but sequential calls are enough for this test.
1240
+ await writer1.markAllSnapshotDone('1/1');
1241
+ await writer1.commit('1/1');
1242
+ await writer1.commit('2/1', { createEmptyCheckpoints: false });
1243
+ const cp2 = await bucketStorage.getCheckpoint();
1244
+ expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
1245
+ await writer2.save({
1246
+ sourceTable,
1247
+ tag: storage.SaveOperationTag.INSERT,
1248
+ after: {
1249
+ id: 'test1',
1250
+ description: 'test1a'
1251
+ },
1252
+ afterReplicaId: test_utils.rid('test1')
1253
+ });
1254
+ // This simulates what happens on a snapshot processor.
1255
+ // This may later change to a flush() rather than commit().
1256
+ await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
1257
+ const cp3 = await bucketStorage.getCheckpoint();
1258
+ expect(cp3.lsn).toEqual('1/1'); // Still unchanged
1259
+ // This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
1260
+ await writer1.commit('4/1', { createEmptyCheckpoints: false });
1261
+ const cp4 = await bucketStorage.getCheckpoint();
1262
+ expect(cp4.lsn).toEqual('4/1');
1566
1263
  });
1567
1264
  test('empty checkpoints (sync rule activation)', async () => {
1568
- const env_22 = { stack: [], error: void 0, hasError: false };
1569
- try {
1570
- const factory = __addDisposableResource(env_22, await generateStorageFactory(), true);
1571
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1265
+ await using factory = await generateStorageFactory();
1266
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1572
1267
  bucket_definitions:
1573
1268
  global:
1574
1269
  data:
1575
1270
  - SELECT id, description FROM "%"
1576
1271
  `, {
1577
- storageVersion
1578
- }));
1579
- const bucketStorage = factory.getInstance(syncRules);
1580
- const writer = __addDisposableResource(env_22, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1581
- const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
1582
- expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1583
- // Snapshot is only valid once we reach 3/1
1584
- await writer.markAllSnapshotDone('3/1');
1585
- // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
1586
- const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
1587
- expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1588
- // No empty checkpoint should be created by the commit above.
1589
- const cp1 = await bucketStorage.getCheckpoint();
1590
- expect(cp1.lsn).toEqual(null);
1591
- // After this commit, the snapshot should be valid.
1592
- // We specifically check that this is done even if createEmptyCheckpoints: false.
1593
- const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
1594
- expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
1595
- // Now, the checkpoint should advance the sync rules active.
1596
- const cp2 = await bucketStorage.getCheckpoint();
1597
- expect(cp2.lsn).toEqual('3/1');
1598
- const activeSyncRules = await factory.getActiveSyncRulesContent();
1599
- expect(activeSyncRules?.id).toEqual(syncRules.id);
1600
- // At this point, it should be a truely empty checkpoint
1601
- const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
1602
- expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
1603
- // Unchanged
1604
- const cp3 = await bucketStorage.getCheckpoint();
1605
- expect(cp3.lsn).toEqual('3/1');
1606
- }
1607
- catch (e_22) {
1608
- env_22.error = e_22;
1609
- env_22.hasError = true;
1610
- }
1611
- finally {
1612
- const result_22 = __disposeResources(env_22);
1613
- if (result_22)
1614
- await result_22;
1615
- }
1272
+ storageVersion
1273
+ }));
1274
+ const bucketStorage = factory.getInstance(syncRules);
1275
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1276
+ const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
1277
+ expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1278
+ // Snapshot is only valid once we reach 3/1
1279
+ await writer.markAllSnapshotDone('3/1');
1280
+ // 2/1 < 3/1 - snapshot not valid yet, block checkpoint
1281
+ const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
1282
+ expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
1283
+ // No empty checkpoint should be created by the commit above.
1284
+ const cp1 = await bucketStorage.getCheckpoint();
1285
+ expect(cp1.lsn).toEqual(null);
1286
+ // After this commit, the snapshot should be valid.
1287
+ // We specifically check that this is done even if createEmptyCheckpoints: false.
1288
+ const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
1289
+ expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
1290
+ // Now, the checkpoint should advance the sync rules active.
1291
+ const cp2 = await bucketStorage.getCheckpoint();
1292
+ expect(cp2.lsn).toEqual('3/1');
1293
+ const activeSyncRules = await factory.getActiveSyncRulesContent();
1294
+ expect(activeSyncRules?.id).toEqual(syncRules.id);
1295
+ // At this point, it should be a truely empty checkpoint
1296
+ const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
1297
+ expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
1298
+ // Unchanged
1299
+ const cp3 = await bucketStorage.getCheckpoint();
1300
+ expect(cp3.lsn).toEqual('3/1');
1616
1301
  });
1617
1302
  test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
1618
- const env_23 = { stack: [], error: void 0, hasError: false };
1619
- try {
1620
- const factory = __addDisposableResource(env_23, await generateStorageFactory(), true);
1621
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1303
+ await using factory = await generateStorageFactory();
1304
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1622
1305
  bucket_definitions:
1623
1306
  global:
1624
1307
  data:
1625
1308
  - SELECT id, description FROM "%"
1626
1309
  `, {
1627
- storageVersion
1628
- }));
1629
- const bucketStorage = factory.getInstance(syncRules);
1630
- const snapshotWriter = __addDisposableResource(env_23, await bucketStorage.createWriter({
1631
- ...test_utils.BATCH_OPTIONS,
1632
- skipExistingRows: true
1633
- }), true);
1634
- const streamingWriter = __addDisposableResource(env_23, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1635
- const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
1636
- const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
1637
- // We simulate two concurrent batches; separate writers are enough for this test.
1638
- // For this test, we assume that we start with a row "test1", which is picked up by a snapshot
1639
- // query, right before the delete is streamed. But the snapshot query is only persisted _after_
1640
- // the delete is streamed, and we need to ensure that the streamed delete takes precedence.
1641
- await streamingWriter.save({
1642
- sourceTable: streamingTable,
1643
- tag: storage.SaveOperationTag.DELETE,
1644
- before: {
1645
- id: 'test1'
1646
- },
1647
- beforeReplicaId: test_utils.rid('test1')
1648
- });
1649
- await streamingWriter.commit('2/1');
1650
- await snapshotWriter.save({
1651
- sourceTable: snapshotTable,
1652
- tag: storage.SaveOperationTag.INSERT,
1653
- after: {
1654
- id: 'test1',
1655
- description: 'test1a'
1656
- },
1657
- afterReplicaId: test_utils.rid('test1')
1658
- });
1659
- await snapshotWriter.markAllSnapshotDone('3/1');
1660
- await snapshotWriter.commit('1/1');
1661
- await streamingWriter.keepalive('3/1');
1662
- const cp = await bucketStorage.getCheckpoint();
1663
- expect(cp.lsn).toEqual('3/1');
1664
- const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]));
1665
- expect(data).toEqual([]);
1666
- }
1667
- catch (e_23) {
1668
- env_23.error = e_23;
1669
- env_23.hasError = true;
1670
- }
1671
- finally {
1672
- const result_23 = __disposeResources(env_23);
1673
- if (result_23)
1674
- await result_23;
1675
- }
1310
+ storageVersion
1311
+ }));
1312
+ const bucketStorage = factory.getInstance(syncRules);
1313
+ await using snapshotWriter = await bucketStorage.createWriter({
1314
+ ...test_utils.BATCH_OPTIONS,
1315
+ skipExistingRows: true
1316
+ });
1317
+ await using streamingWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1318
+ const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
1319
+ const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
1320
+ // We simulate two concurrent batches; separate writers are enough for this test.
1321
+ // For this test, we assume that we start with a row "test1", which is picked up by a snapshot
1322
+ // query, right before the delete is streamed. But the snapshot query is only persisted _after_
1323
+ // the delete is streamed, and we need to ensure that the streamed delete takes precedence.
1324
+ await streamingWriter.save({
1325
+ sourceTable: streamingTable,
1326
+ tag: storage.SaveOperationTag.DELETE,
1327
+ before: {
1328
+ id: 'test1'
1329
+ },
1330
+ beforeReplicaId: test_utils.rid('test1')
1331
+ });
1332
+ await streamingWriter.commit('2/1');
1333
+ await snapshotWriter.save({
1334
+ sourceTable: snapshotTable,
1335
+ tag: storage.SaveOperationTag.INSERT,
1336
+ after: {
1337
+ id: 'test1',
1338
+ description: 'test1a'
1339
+ },
1340
+ afterReplicaId: test_utils.rid('test1')
1341
+ });
1342
+ await snapshotWriter.markAllSnapshotDone('3/1');
1343
+ await snapshotWriter.commit('1/1');
1344
+ await streamingWriter.keepalive('3/1');
1345
+ const cp = await bucketStorage.getCheckpoint();
1346
+ expect(cp.lsn).toEqual('3/1');
1347
+ const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]));
1348
+ expect(data).toEqual([]);
1676
1349
  });
1677
1350
  }
1678
1351
  /**
@@ -1683,62 +1356,50 @@ bucket_definitions:
1683
1356
  export function testChecksumBatching(config) {
1684
1357
  const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
1685
1358
  test('checksums for multiple buckets', async () => {
1686
- const env_24 = { stack: [], error: void 0, hasError: false };
1687
- try {
1688
- const factory = __addDisposableResource(env_24, await config.factory(), true);
1689
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1359
+ await using factory = await config.factory();
1360
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
1690
1361
  bucket_definitions:
1691
1362
  user:
1692
1363
  parameters: select request.user_id() as user_id
1693
1364
  data:
1694
1365
  - select id, description from test where user_id = bucket.user_id
1695
1366
  `, {
1696
- storageVersion
1697
- }));
1698
- const bucketStorage = factory.getInstance(syncRules);
1699
- const writer = __addDisposableResource(env_24, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1700
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1701
- await writer.markAllSnapshotDone('1/1');
1702
- for (let u of ['u1', 'u2', 'u3', 'u4']) {
1703
- for (let t of ['t1', 't2', 't3', 't4']) {
1704
- const id = `${t}_${u}`;
1705
- await writer.save({
1706
- sourceTable,
1707
- tag: storage.SaveOperationTag.INSERT,
1708
- after: {
1709
- id,
1710
- description: `${t} description`,
1711
- user_id: u
1712
- },
1713
- afterReplicaId: test_utils.rid(id)
1714
- });
1715
- }
1367
+ storageVersion
1368
+ }));
1369
+ const bucketStorage = factory.getInstance(syncRules);
1370
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1371
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1372
+ await writer.markAllSnapshotDone('1/1');
1373
+ for (let u of ['u1', 'u2', 'u3', 'u4']) {
1374
+ for (let t of ['t1', 't2', 't3', 't4']) {
1375
+ const id = `${t}_${u}`;
1376
+ await writer.save({
1377
+ sourceTable,
1378
+ tag: storage.SaveOperationTag.INSERT,
1379
+ after: {
1380
+ id,
1381
+ description: `${t} description`,
1382
+ user_id: u
1383
+ },
1384
+ afterReplicaId: test_utils.rid(id)
1385
+ });
1716
1386
  }
1717
- await writer.commit('1/1');
1718
- const { checkpoint } = await bucketStorage.getCheckpoint();
1719
- bucketStorage.clearChecksumCache();
1720
- const users = ['u1', 'u2', 'u3', 'u4'];
1721
- const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
1722
- const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
1723
- const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
1724
- checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
1725
- const expected = bucketRequests.map((request, index) => ({
1726
- bucket: request.bucket,
1727
- count: 4,
1728
- checksum: expectedChecksums[index]
1729
- }));
1730
- expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
1731
- expect(checksums).toEqual(expected);
1732
- }
1733
- catch (e_24) {
1734
- env_24.error = e_24;
1735
- env_24.hasError = true;
1736
- }
1737
- finally {
1738
- const result_24 = __disposeResources(env_24);
1739
- if (result_24)
1740
- await result_24;
1741
1387
  }
1388
+ await writer.commit('1/1');
1389
+ const { checkpoint } = await bucketStorage.getCheckpoint();
1390
+ bucketStorage.clearChecksumCache();
1391
+ const users = ['u1', 'u2', 'u3', 'u4'];
1392
+ const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
1393
+ const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
1394
+ const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
1395
+ checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
1396
+ const expected = bucketRequests.map((request, index) => ({
1397
+ bucket: request.bucket,
1398
+ count: 4,
1399
+ checksum: expectedChecksums[index]
1400
+ }));
1401
+ expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
1402
+ expect(checksums).toEqual(expected);
1742
1403
  });
1743
1404
  }
1744
1405
  //# sourceMappingURL=register-data-storage-data-tests.js.map