@powersync/service-core-tests 0.15.2 → 0.15.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,55 +1,3 @@
1
- var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
- if (value !== null && value !== void 0) {
3
- if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
- var dispose, inner;
5
- if (async) {
6
- if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
- dispose = value[Symbol.asyncDispose];
8
- }
9
- if (dispose === void 0) {
10
- if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
- dispose = value[Symbol.dispose];
12
- if (async) inner = dispose;
13
- }
14
- if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
- if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
- env.stack.push({ value: value, dispose: dispose, async: async });
17
- }
18
- else if (async) {
19
- env.stack.push({ async: true });
20
- }
21
- return value;
22
- };
23
- var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
- return function (env) {
25
- function fail(e) {
26
- env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
- env.hasError = true;
28
- }
29
- var r, s = 0;
30
- function next() {
31
- while (r = env.stack.pop()) {
32
- try {
33
- if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
- if (r.dispose) {
35
- var result = r.dispose.call(r.value);
36
- if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
- }
38
- else s |= 1;
39
- }
40
- catch (e) {
41
- fail(e);
42
- }
43
- }
44
- if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
- if (env.hasError) throw env.error;
46
- }
47
- return next();
48
- };
49
- })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
- var e = new Error(message);
51
- return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
- });
53
1
  import { createCoreAPIMetrics, JwtPayload, storage, sync, updateSyncRulesFromYaml } from '@powersync/service-core';
54
2
  import { JSONBig } from '@powersync/service-jsonbig';
55
3
  import path from 'path';
@@ -94,67 +42,53 @@ export function registerSyncTests(configOrFactory, options = {}) {
94
42
  }));
95
43
  };
96
44
  test('sync global data', async () => {
97
- const env_1 = { stack: [], error: void 0, hasError: false };
98
- try {
99
- const f = __addDisposableResource(env_1, await factory(), true);
100
- const syncRules = await updateSyncRules(f, {
101
- content: BASIC_SYNC_RULES
102
- });
103
- const bucketStorage = f.getInstance(syncRules);
104
- const writer = __addDisposableResource(env_1, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
105
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
106
- await writer.markAllSnapshotDone('0/1');
107
- await writer.save({
108
- sourceTable,
109
- tag: storage.SaveOperationTag.INSERT,
110
- after: {
111
- id: 't1',
112
- description: 'Test 1'
113
- },
114
- afterReplicaId: 't1'
115
- });
116
- await writer.save({
117
- sourceTable,
118
- tag: storage.SaveOperationTag.INSERT,
119
- after: {
120
- id: 't2',
121
- description: 'Test 2'
122
- },
123
- afterReplicaId: 't2'
124
- });
125
- await writer.commit('0/1');
126
- const stream = sync.streamResponse({
127
- syncContext,
128
- bucketStorage: bucketStorage,
129
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
130
- params: {
131
- buckets: [],
132
- include_checksum: true,
133
- raw_data: true
134
- },
135
- tracker,
136
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
137
- isEncodingAsBson: false
138
- });
139
- const lines = await consumeCheckpointLines(stream);
140
- expect(lines).toMatchSnapshot();
141
- }
142
- catch (e_1) {
143
- env_1.error = e_1;
144
- env_1.hasError = true;
145
- }
146
- finally {
147
- const result_1 = __disposeResources(env_1);
148
- if (result_1)
149
- await result_1;
150
- }
45
+ await using f = await factory();
46
+ const syncRules = await updateSyncRules(f, {
47
+ content: BASIC_SYNC_RULES
48
+ });
49
+ const bucketStorage = f.getInstance(syncRules);
50
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
51
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
52
+ await writer.markAllSnapshotDone('0/1');
53
+ await writer.save({
54
+ sourceTable,
55
+ tag: storage.SaveOperationTag.INSERT,
56
+ after: {
57
+ id: 't1',
58
+ description: 'Test 1'
59
+ },
60
+ afterReplicaId: 't1'
61
+ });
62
+ await writer.save({
63
+ sourceTable,
64
+ tag: storage.SaveOperationTag.INSERT,
65
+ after: {
66
+ id: 't2',
67
+ description: 'Test 2'
68
+ },
69
+ afterReplicaId: 't2'
70
+ });
71
+ await writer.commit('0/1');
72
+ const stream = sync.streamResponse({
73
+ syncContext,
74
+ bucketStorage: bucketStorage,
75
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
76
+ params: {
77
+ buckets: [],
78
+ include_checksum: true,
79
+ raw_data: true
80
+ },
81
+ tracker,
82
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
83
+ isEncodingAsBson: false
84
+ });
85
+ const lines = await consumeCheckpointLines(stream);
86
+ expect(lines).toMatchSnapshot();
151
87
  });
152
88
  test('sync buckets in order', async () => {
153
- const env_2 = { stack: [], error: void 0, hasError: false };
154
- try {
155
- const f = __addDisposableResource(env_2, await factory(), true);
156
- const syncRules = await updateSyncRules(f, {
157
- content: `
89
+ await using f = await factory();
90
+ const syncRules = await updateSyncRules(f, {
91
+ content: `
158
92
  bucket_definitions:
159
93
  b0:
160
94
  priority: 2
@@ -165,62 +99,50 @@ bucket_definitions:
165
99
  data:
166
100
  - SELECT * FROM test WHERE LENGTH(id) > 2;
167
101
  `
168
- });
169
- const bucketStorage = f.getInstance(syncRules);
170
- const writer = __addDisposableResource(env_2, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
171
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
172
- await writer.markAllSnapshotDone('0/1');
173
- await writer.save({
174
- sourceTable: testTable,
175
- tag: storage.SaveOperationTag.INSERT,
176
- after: {
177
- id: 't1',
178
- description: 'Test 1'
179
- },
180
- afterReplicaId: 't1'
181
- });
182
- await writer.save({
183
- sourceTable: testTable,
184
- tag: storage.SaveOperationTag.INSERT,
185
- after: {
186
- id: 'earlier',
187
- description: 'Test 2'
188
- },
189
- afterReplicaId: 'earlier'
190
- });
191
- await writer.commit('0/1');
192
- const stream = sync.streamResponse({
193
- syncContext,
194
- bucketStorage,
195
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
196
- params: {
197
- buckets: [],
198
- include_checksum: true,
199
- raw_data: true
200
- },
201
- tracker,
202
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
203
- isEncodingAsBson: false
204
- });
205
- const lines = await consumeCheckpointLines(stream);
206
- expect(lines).toMatchSnapshot();
207
- }
208
- catch (e_2) {
209
- env_2.error = e_2;
210
- env_2.hasError = true;
211
- }
212
- finally {
213
- const result_2 = __disposeResources(env_2);
214
- if (result_2)
215
- await result_2;
216
- }
102
+ });
103
+ const bucketStorage = f.getInstance(syncRules);
104
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
105
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
106
+ await writer.markAllSnapshotDone('0/1');
107
+ await writer.save({
108
+ sourceTable: testTable,
109
+ tag: storage.SaveOperationTag.INSERT,
110
+ after: {
111
+ id: 't1',
112
+ description: 'Test 1'
113
+ },
114
+ afterReplicaId: 't1'
115
+ });
116
+ await writer.save({
117
+ sourceTable: testTable,
118
+ tag: storage.SaveOperationTag.INSERT,
119
+ after: {
120
+ id: 'earlier',
121
+ description: 'Test 2'
122
+ },
123
+ afterReplicaId: 'earlier'
124
+ });
125
+ await writer.commit('0/1');
126
+ const stream = sync.streamResponse({
127
+ syncContext,
128
+ bucketStorage,
129
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
130
+ params: {
131
+ buckets: [],
132
+ include_checksum: true,
133
+ raw_data: true
134
+ },
135
+ tracker,
136
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
137
+ isEncodingAsBson: false
138
+ });
139
+ const lines = await consumeCheckpointLines(stream);
140
+ expect(lines).toMatchSnapshot();
217
141
  });
218
142
  test('sync interrupts low-priority buckets on new checkpoints', async () => {
219
- const env_3 = { stack: [], error: void 0, hasError: false };
220
- try {
221
- const f = __addDisposableResource(env_3, await factory(), true);
222
- const syncRules = await updateSyncRules(f, {
223
- content: `
143
+ await using f = await factory();
144
+ const syncRules = await updateSyncRules(f, {
145
+ content: `
224
146
  bucket_definitions:
225
147
  b0:
226
148
  priority: 2
@@ -231,105 +153,93 @@ bucket_definitions:
231
153
  data:
232
154
  - SELECT * FROM test WHERE LENGTH(id) > 5;
233
155
  `
234
- });
235
- const bucketStorage = f.getInstance(syncRules);
236
- const writer = __addDisposableResource(env_3, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
237
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
238
- await writer.markAllSnapshotDone('0/1');
239
- // Initial data: Add one priority row and 10k low-priority rows.
156
+ });
157
+ const bucketStorage = f.getInstance(syncRules);
158
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
159
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
160
+ await writer.markAllSnapshotDone('0/1');
161
+ // Initial data: Add one priority row and 10k low-priority rows.
162
+ await writer.save({
163
+ sourceTable: testTable,
164
+ tag: storage.SaveOperationTag.INSERT,
165
+ after: {
166
+ id: 'highprio',
167
+ description: 'High priority row'
168
+ },
169
+ afterReplicaId: 'highprio'
170
+ });
171
+ for (let i = 0; i < 10_000; i++) {
240
172
  await writer.save({
241
173
  sourceTable: testTable,
242
174
  tag: storage.SaveOperationTag.INSERT,
243
175
  after: {
244
- id: 'highprio',
245
- description: 'High priority row'
176
+ id: `${i}`,
177
+ description: 'low prio'
246
178
  },
247
- afterReplicaId: 'highprio'
179
+ afterReplicaId: `${i}`
248
180
  });
249
- for (let i = 0; i < 10_000; i++) {
250
- await writer.save({
251
- sourceTable: testTable,
252
- tag: storage.SaveOperationTag.INSERT,
253
- after: {
254
- id: `${i}`,
255
- description: 'low prio'
256
- },
257
- afterReplicaId: `${i}`
258
- });
181
+ }
182
+ await writer.commit('0/1');
183
+ const stream = sync.streamResponse({
184
+ syncContext,
185
+ bucketStorage,
186
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
187
+ params: {
188
+ buckets: [],
189
+ include_checksum: true,
190
+ raw_data: true
191
+ },
192
+ tracker,
193
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
194
+ isEncodingAsBson: false
195
+ });
196
+ let sentCheckpoints = 0;
197
+ let sentRows = 0;
198
+ for await (let next of stream) {
199
+ if (typeof next == 'string') {
200
+ next = JSON.parse(next);
259
201
  }
260
- await writer.commit('0/1');
261
- const stream = sync.streamResponse({
262
- syncContext,
263
- bucketStorage,
264
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
265
- params: {
266
- buckets: [],
267
- include_checksum: true,
268
- raw_data: true
269
- },
270
- tracker,
271
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
272
- isEncodingAsBson: false
273
- });
274
- let sentCheckpoints = 0;
275
- let sentRows = 0;
276
- for await (let next of stream) {
277
- if (typeof next == 'string') {
278
- next = JSON.parse(next);
279
- }
280
- if (typeof next === 'object' && next !== null) {
281
- if ('partial_checkpoint_complete' in next) {
282
- if (sentCheckpoints == 1) {
283
- // Save new data to interrupt the low-priority sync.
284
- // Add another high-priority row. This should interrupt the long-running low-priority sync.
285
- await writer.save({
286
- sourceTable: testTable,
287
- tag: storage.SaveOperationTag.INSERT,
288
- after: {
289
- id: 'highprio2',
290
- description: 'Another high-priority row'
291
- },
292
- afterReplicaId: 'highprio2'
293
- });
294
- await writer.commit('0/2');
295
- }
296
- else {
297
- // Low-priority sync from the first checkpoint was interrupted. This should not happen before
298
- // 1000 low-priority items were synchronized.
299
- expect(sentCheckpoints).toBe(2);
300
- expect(sentRows).toBeGreaterThan(1000);
301
- }
302
- }
303
- if ('checkpoint' in next || 'checkpoint_diff' in next) {
304
- sentCheckpoints += 1;
202
+ if (typeof next === 'object' && next !== null) {
203
+ if ('partial_checkpoint_complete' in next) {
204
+ if (sentCheckpoints == 1) {
205
+ // Save new data to interrupt the low-priority sync.
206
+ // Add another high-priority row. This should interrupt the long-running low-priority sync.
207
+ await writer.save({
208
+ sourceTable: testTable,
209
+ tag: storage.SaveOperationTag.INSERT,
210
+ after: {
211
+ id: 'highprio2',
212
+ description: 'Another high-priority row'
213
+ },
214
+ afterReplicaId: 'highprio2'
215
+ });
216
+ await writer.commit('0/2');
305
217
  }
306
- if ('data' in next) {
307
- sentRows += next.data.data.length;
308
- }
309
- if ('checkpoint_complete' in next) {
310
- break;
218
+ else {
219
+ // Low-priority sync from the first checkpoint was interrupted. This should not happen before
220
+ // 1000 low-priority items were synchronized.
221
+ expect(sentCheckpoints).toBe(2);
222
+ expect(sentRows).toBeGreaterThan(1000);
311
223
  }
312
224
  }
225
+ if ('checkpoint' in next || 'checkpoint_diff' in next) {
226
+ sentCheckpoints += 1;
227
+ }
228
+ if ('data' in next) {
229
+ sentRows += next.data.data.length;
230
+ }
231
+ if ('checkpoint_complete' in next) {
232
+ break;
233
+ }
313
234
  }
314
- expect(sentCheckpoints).toBe(2);
315
- expect(sentRows).toBe(10002);
316
- }
317
- catch (e_3) {
318
- env_3.error = e_3;
319
- env_3.hasError = true;
320
- }
321
- finally {
322
- const result_3 = __disposeResources(env_3);
323
- if (result_3)
324
- await result_3;
325
235
  }
236
+ expect(sentCheckpoints).toBe(2);
237
+ expect(sentRows).toBe(10002);
326
238
  });
327
239
  test('sync interruptions with unrelated data', async () => {
328
- const env_4 = { stack: [], error: void 0, hasError: false };
329
- try {
330
- const f = __addDisposableResource(env_4, await factory(), true);
331
- const syncRules = await updateSyncRules(f, {
332
- content: `
240
+ await using f = await factory();
241
+ const syncRules = await updateSyncRules(f, {
242
+ content: `
333
243
  bucket_definitions:
334
244
  b0:
335
245
  priority: 2
@@ -341,128 +251,116 @@ bucket_definitions:
341
251
  data:
342
252
  - SELECT * FROM test WHERE LENGTH(id) > 5 AND description = bucket.user_id;
343
253
  `
344
- });
345
- const bucketStorage = f.getInstance(syncRules);
346
- const writer = __addDisposableResource(env_4, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
347
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
348
- await writer.markAllSnapshotDone('0/1');
349
- // Initial data: Add one priority row and 10k low-priority rows.
254
+ });
255
+ const bucketStorage = f.getInstance(syncRules);
256
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
257
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
258
+ await writer.markAllSnapshotDone('0/1');
259
+ // Initial data: Add one priority row and 10k low-priority rows.
260
+ await writer.save({
261
+ sourceTable: testTable,
262
+ tag: storage.SaveOperationTag.INSERT,
263
+ after: {
264
+ id: 'highprio',
265
+ description: 'user_one'
266
+ },
267
+ afterReplicaId: 'highprio'
268
+ });
269
+ for (let i = 0; i < 10_000; i++) {
350
270
  await writer.save({
351
271
  sourceTable: testTable,
352
272
  tag: storage.SaveOperationTag.INSERT,
353
273
  after: {
354
- id: 'highprio',
355
- description: 'user_one'
274
+ id: `${i}`,
275
+ description: 'low prio'
356
276
  },
357
- afterReplicaId: 'highprio'
277
+ afterReplicaId: `${i}`
358
278
  });
359
- for (let i = 0; i < 10_000; i++) {
360
- await writer.save({
361
- sourceTable: testTable,
362
- tag: storage.SaveOperationTag.INSERT,
363
- after: {
364
- id: `${i}`,
365
- description: 'low prio'
366
- },
367
- afterReplicaId: `${i}`
368
- });
279
+ }
280
+ await writer.commit('0/1');
281
+ const stream = sync.streamResponse({
282
+ syncContext,
283
+ bucketStorage,
284
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
285
+ params: {
286
+ buckets: [],
287
+ include_checksum: true,
288
+ raw_data: true
289
+ },
290
+ tracker,
291
+ token: new JwtPayload({ sub: 'user_one', exp: Date.now() / 1000 + 100000 }),
292
+ isEncodingAsBson: false
293
+ });
294
+ let sentCheckpoints = 0;
295
+ let completedCheckpoints = 0;
296
+ let sentRows = 0;
297
+ // Expected flow:
298
+ // 1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
299
+ // 2. We insert a new row that is not part of a bucket relevant to this stream.
300
+ // 3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
301
+ // 4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
302
+ for await (let next of stream) {
303
+ if (typeof next == 'string') {
304
+ next = JSON.parse(next);
369
305
  }
370
- await writer.commit('0/1');
371
- const stream = sync.streamResponse({
372
- syncContext,
373
- bucketStorage,
374
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
375
- params: {
376
- buckets: [],
377
- include_checksum: true,
378
- raw_data: true
379
- },
380
- tracker,
381
- token: new JwtPayload({ sub: 'user_one', exp: Date.now() / 1000 + 100000 }),
382
- isEncodingAsBson: false
383
- });
384
- let sentCheckpoints = 0;
385
- let completedCheckpoints = 0;
386
- let sentRows = 0;
387
- // Expected flow:
388
- // 1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
389
- // 2. We insert a new row that is not part of a bucket relevant to this stream.
390
- // 3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
391
- // 4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
392
- for await (let next of stream) {
393
- if (typeof next == 'string') {
394
- next = JSON.parse(next);
395
- }
396
- if (typeof next === 'object' && next !== null) {
397
- if ('partial_checkpoint_complete' in next) {
398
- if (sentCheckpoints == 1) {
399
- // Add a high-priority row that doesn't affect this sync stream.
400
- await writer.save({
401
- sourceTable: testTable,
402
- tag: storage.SaveOperationTag.INSERT,
403
- after: {
404
- id: 'highprio2',
405
- description: 'user_two'
406
- },
407
- afterReplicaId: 'highprio2'
408
- });
409
- await writer.commit('0/2');
410
- }
411
- else {
412
- expect(sentCheckpoints).toBe(2);
413
- expect(sentRows).toBe(10002);
414
- }
306
+ if (typeof next === 'object' && next !== null) {
307
+ if ('partial_checkpoint_complete' in next) {
308
+ if (sentCheckpoints == 1) {
309
+ // Add a high-priority row that doesn't affect this sync stream.
310
+ await writer.save({
311
+ sourceTable: testTable,
312
+ tag: storage.SaveOperationTag.INSERT,
313
+ after: {
314
+ id: 'highprio2',
315
+ description: 'user_two'
316
+ },
317
+ afterReplicaId: 'highprio2'
318
+ });
319
+ await writer.commit('0/2');
415
320
  }
416
- if ('checkpoint' in next || 'checkpoint_diff' in next) {
417
- sentCheckpoints += 1;
321
+ else {
322
+ expect(sentCheckpoints).toBe(2);
323
+ expect(sentRows).toBe(10002);
418
324
  }
419
- if ('data' in next) {
420
- sentRows += next.data.data.length;
325
+ }
326
+ if ('checkpoint' in next || 'checkpoint_diff' in next) {
327
+ sentCheckpoints += 1;
328
+ }
329
+ if ('data' in next) {
330
+ sentRows += next.data.data.length;
331
+ }
332
+ if ('checkpoint_complete' in next) {
333
+ completedCheckpoints++;
334
+ if (completedCheckpoints == 2) {
335
+ break;
421
336
  }
422
- if ('checkpoint_complete' in next) {
423
- completedCheckpoints++;
424
- if (completedCheckpoints == 2) {
425
- break;
426
- }
427
- if (completedCheckpoints == 1) {
428
- expect(sentRows).toBe(10001);
429
- // Add a high-priority row that affects this sync stream.
430
- await writer.save({
431
- sourceTable: testTable,
432
- tag: storage.SaveOperationTag.INSERT,
433
- after: {
434
- id: 'highprio3',
435
- description: 'user_one'
436
- },
437
- afterReplicaId: 'highprio3'
438
- });
439
- await writer.commit('0/3');
440
- }
337
+ if (completedCheckpoints == 1) {
338
+ expect(sentRows).toBe(10001);
339
+ // Add a high-priority row that affects this sync stream.
340
+ await writer.save({
341
+ sourceTable: testTable,
342
+ tag: storage.SaveOperationTag.INSERT,
343
+ after: {
344
+ id: 'highprio3',
345
+ description: 'user_one'
346
+ },
347
+ afterReplicaId: 'highprio3'
348
+ });
349
+ await writer.commit('0/3');
441
350
  }
442
351
  }
443
352
  }
444
- expect(sentCheckpoints).toBe(2);
445
- expect(sentRows).toBe(10002);
446
- }
447
- catch (e_4) {
448
- env_4.error = e_4;
449
- env_4.hasError = true;
450
- }
451
- finally {
452
- const result_4 = __disposeResources(env_4);
453
- if (result_4)
454
- await result_4;
455
353
  }
354
+ expect(sentCheckpoints).toBe(2);
355
+ expect(sentRows).toBe(10002);
456
356
  });
457
357
  test('sync interrupts low-priority buckets on new checkpoints (2)', async () => {
458
- const env_5 = { stack: [], error: void 0, hasError: false };
459
- try {
460
- const f = __addDisposableResource(env_5, await factory(), true);
461
- // bucket0a -> send all data
462
- // then interrupt checkpoint with new data for all buckets
463
- // -> data for all buckets should be sent in the new checkpoint
464
- const syncRules = await updateSyncRules(f, {
465
- content: `
358
+ await using f = await factory();
359
+ // bucket0a -> send all data
360
+ // then interrupt checkpoint with new data for all buckets
361
+ // -> data for all buckets should be sent in the new checkpoint
362
+ const syncRules = await updateSyncRules(f, {
363
+ content: `
466
364
  bucket_definitions:
467
365
  b0a:
468
366
  priority: 2
@@ -477,613 +375,701 @@ bucket_definitions:
477
375
  data:
478
376
  - SELECT * FROM test WHERE LENGTH(id) > 5;
479
377
  `
480
- });
481
- const bucketStorage = f.getInstance(syncRules);
482
- const writer = __addDisposableResource(env_5, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
483
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
484
- await writer.markAllSnapshotDone('0/1');
485
- // Initial data: Add one priority row and 10k low-priority rows.
378
+ });
379
+ const bucketStorage = f.getInstance(syncRules);
380
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
381
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
382
+ await writer.markAllSnapshotDone('0/1');
383
+ // Initial data: Add one priority row and 10k low-priority rows.
384
+ await writer.save({
385
+ sourceTable: testTable,
386
+ tag: storage.SaveOperationTag.INSERT,
387
+ after: {
388
+ id: 'highprio',
389
+ description: 'High priority row'
390
+ },
391
+ afterReplicaId: 'highprio'
392
+ });
393
+ for (let i = 0; i < 2_000; i++) {
486
394
  await writer.save({
487
395
  sourceTable: testTable,
488
396
  tag: storage.SaveOperationTag.INSERT,
489
397
  after: {
490
- id: 'highprio',
491
- description: 'High priority row'
398
+ id: `${i}`,
399
+ description: 'low prio'
492
400
  },
493
- afterReplicaId: 'highprio'
401
+ afterReplicaId: `${i}`
494
402
  });
495
- for (let i = 0; i < 2_000; i++) {
496
- await writer.save({
497
- sourceTable: testTable,
498
- tag: storage.SaveOperationTag.INSERT,
499
- after: {
500
- id: `${i}`,
501
- description: 'low prio'
502
- },
503
- afterReplicaId: `${i}`
504
- });
403
+ }
404
+ await writer.commit('0/1');
405
+ const stream = sync.streamResponse({
406
+ syncContext,
407
+ bucketStorage,
408
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
409
+ params: {
410
+ buckets: [],
411
+ include_checksum: true,
412
+ raw_data: true
413
+ },
414
+ tracker,
415
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
416
+ isEncodingAsBson: false
417
+ });
418
+ let sentRows = 0;
419
+ let lines = [];
420
+ for await (let next of stream) {
421
+ if (typeof next == 'string') {
422
+ next = JSON.parse(next);
505
423
  }
506
- await writer.commit('0/1');
507
- const stream = sync.streamResponse({
508
- syncContext,
509
- bucketStorage,
510
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
511
- params: {
512
- buckets: [],
513
- include_checksum: true,
514
- raw_data: true
515
- },
516
- tracker,
517
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
518
- isEncodingAsBson: false
519
- });
520
- let sentRows = 0;
521
- let lines = [];
522
- for await (let next of stream) {
523
- if (typeof next == 'string') {
524
- next = JSON.parse(next);
424
+ if (typeof next === 'object' && next !== null) {
425
+ if ('partial_checkpoint_complete' in next) {
426
+ lines.push(next);
525
427
  }
526
- if (typeof next === 'object' && next !== null) {
527
- if ('partial_checkpoint_complete' in next) {
528
- lines.push(next);
529
- }
530
- if ('checkpoint' in next || 'checkpoint_diff' in next) {
531
- lines.push(next);
532
- }
533
- if ('data' in next) {
534
- lines.push({ data: { ...next.data, data: undefined } });
535
- sentRows += next.data.data.length;
536
- if (sentRows == 1001) {
537
- // Save new data to interrupt the low-priority sync.
538
- // Add another high-priority row. This should interrupt the long-running low-priority sync.
539
- await writer.save({
540
- sourceTable: testTable,
541
- tag: storage.SaveOperationTag.INSERT,
542
- after: {
543
- id: 'highprio2',
544
- description: 'Another high-priority row'
545
- },
546
- afterReplicaId: 'highprio2'
547
- });
548
- // Also add a low-priority row
549
- await writer.save({
550
- sourceTable: testTable,
551
- tag: storage.SaveOperationTag.INSERT,
552
- after: {
553
- id: '2001',
554
- description: 'Another low-priority row'
555
- },
556
- afterReplicaId: '2001'
557
- });
558
- await writer.commit('0/2');
559
- }
560
- if (sentRows >= 1000 && sentRows <= 2001) {
561
- // pause for a bit to give the stream time to process interruptions.
562
- // This covers the data batch above and the next one.
563
- await timers.setTimeout(50);
564
- }
428
+ if ('checkpoint' in next || 'checkpoint_diff' in next) {
429
+ lines.push(next);
430
+ }
431
+ if ('data' in next) {
432
+ lines.push({ data: { ...next.data, data: undefined } });
433
+ sentRows += next.data.data.length;
434
+ if (sentRows == 1001) {
435
+ // Save new data to interrupt the low-priority sync.
436
+ // Add another high-priority row. This should interrupt the long-running low-priority sync.
437
+ await writer.save({
438
+ sourceTable: testTable,
439
+ tag: storage.SaveOperationTag.INSERT,
440
+ after: {
441
+ id: 'highprio2',
442
+ description: 'Another high-priority row'
443
+ },
444
+ afterReplicaId: 'highprio2'
445
+ });
446
+ // Also add a low-priority row
447
+ await writer.save({
448
+ sourceTable: testTable,
449
+ tag: storage.SaveOperationTag.INSERT,
450
+ after: {
451
+ id: '2001',
452
+ description: 'Another low-priority row'
453
+ },
454
+ afterReplicaId: '2001'
455
+ });
456
+ await writer.commit('0/2');
565
457
  }
566
- if ('checkpoint_complete' in next) {
567
- lines.push(next);
568
- break;
458
+ if (sentRows >= 1000 && sentRows <= 2001) {
459
+ // pause for a bit to give the stream time to process interruptions.
460
+ // This covers the data batch above and the next one.
461
+ await timers.setTimeout(50);
569
462
  }
570
463
  }
464
+ if ('checkpoint_complete' in next) {
465
+ lines.push(next);
466
+ break;
467
+ }
571
468
  }
572
- // Expected lines (full details in snapshot):
573
- //
574
- // checkpoint (4001)
575
- // data (b1[] 0 -> 1)
576
- // partial_checkpoint_complete (4001, priority 1)
577
- // data (b0a[], 0 -> 2000)
578
- // ## adds new data, interrupting the checkpoint
579
- // data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
580
- // checkpoint_diff (4004)
581
- // data (b1[], 1 -> 4002)
582
- // partial_checkpoint_complete (4004, priority 1)
583
- // data (b0a[], 4000 -> 4003)
584
- // data (b0b[], 0 -> 1999)
585
- // data (b0b[], 1999 -> 3999)
586
- // data (b0b[], 3999 -> 4004)
587
- // checkpoint_complete (4004)
588
- expect(lines).toMatchSnapshot();
589
- expect(sentRows).toBe(4004);
590
- }
591
- catch (e_5) {
592
- env_5.error = e_5;
593
- env_5.hasError = true;
594
- }
595
- finally {
596
- const result_5 = __disposeResources(env_5);
597
- if (result_5)
598
- await result_5;
599
469
  }
470
+ // Expected lines (full details in snapshot):
471
+ //
472
+ // checkpoint (4001)
473
+ // data (b1[] 0 -> 1)
474
+ // partial_checkpoint_complete (4001, priority 1)
475
+ // data (b0a[], 0 -> 2000)
476
+ // ## adds new data, interrupting the checkpoint
477
+ // data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
478
+ // checkpoint_diff (4004)
479
+ // data (b1[], 1 -> 4002)
480
+ // partial_checkpoint_complete (4004, priority 1)
481
+ // data (b0a[], 4000 -> 4003)
482
+ // data (b0b[], 0 -> 1999)
483
+ // data (b0b[], 1999 -> 3999)
484
+ // data (b0b[], 3999 -> 4004)
485
+ // checkpoint_complete (4004)
486
+ expect(lines).toMatchSnapshot();
487
+ expect(sentRows).toBe(4004);
600
488
  });
601
489
  test('sends checkpoint complete line for empty checkpoint', async () => {
602
- const env_6 = { stack: [], error: void 0, hasError: false };
603
- try {
604
- const f = __addDisposableResource(env_6, await factory(), true);
605
- const syncRules = await updateSyncRules(f, {
606
- content: BASIC_SYNC_RULES
607
- });
608
- const bucketStorage = f.getInstance(syncRules);
609
- const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
610
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
611
- await writer.markAllSnapshotDone('0/1');
612
- await writer.save({
613
- sourceTable: testTable,
614
- tag: storage.SaveOperationTag.INSERT,
615
- after: {
616
- id: 't1',
617
- description: 'sync'
618
- },
619
- afterReplicaId: 't1'
620
- });
621
- await writer.commit('0/1');
622
- const stream = sync.streamResponse({
623
- syncContext,
624
- bucketStorage,
625
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
626
- params: {
627
- buckets: [],
628
- include_checksum: true,
629
- raw_data: true
630
- },
631
- tracker,
632
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 100000 }),
633
- isEncodingAsBson: false
634
- });
635
- const lines = [];
636
- let receivedCompletions = 0;
637
- for await (let next of stream) {
638
- if (typeof next == 'string') {
639
- next = JSON.parse(next);
640
- }
641
- lines.push(next);
642
- if (typeof next === 'object' && next !== null) {
643
- if ('checkpoint_complete' in next) {
644
- receivedCompletions++;
645
- if (receivedCompletions == 1) {
646
- // Trigger an empty bucket update.
647
- await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } });
648
- await writer.commit('1/0');
649
- }
650
- else {
651
- break;
652
- }
490
+ await using f = await factory();
491
+ const syncRules = await updateSyncRules(f, {
492
+ content: BASIC_SYNC_RULES
493
+ });
494
+ const bucketStorage = f.getInstance(syncRules);
495
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
496
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
497
+ await writer.markAllSnapshotDone('0/1');
498
+ await writer.save({
499
+ sourceTable: testTable,
500
+ tag: storage.SaveOperationTag.INSERT,
501
+ after: {
502
+ id: 't1',
503
+ description: 'sync'
504
+ },
505
+ afterReplicaId: 't1'
506
+ });
507
+ await writer.commit('0/1');
508
+ const stream = sync.streamResponse({
509
+ syncContext,
510
+ bucketStorage,
511
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
512
+ params: {
513
+ buckets: [],
514
+ include_checksum: true,
515
+ raw_data: true
516
+ },
517
+ tracker,
518
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 100000 }),
519
+ isEncodingAsBson: false
520
+ });
521
+ const lines = [];
522
+ let receivedCompletions = 0;
523
+ for await (let next of stream) {
524
+ if (typeof next == 'string') {
525
+ next = JSON.parse(next);
526
+ }
527
+ lines.push(next);
528
+ if (typeof next === 'object' && next !== null) {
529
+ if ('checkpoint_complete' in next) {
530
+ receivedCompletions++;
531
+ if (receivedCompletions == 1) {
532
+ // Trigger an empty bucket update.
533
+ await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } });
534
+ await writer.commit('1/0');
535
+ }
536
+ else {
537
+ break;
653
538
  }
654
539
  }
655
540
  }
656
- expect(lines).toMatchSnapshot();
657
- }
658
- catch (e_6) {
659
- env_6.error = e_6;
660
- env_6.hasError = true;
661
- }
662
- finally {
663
- const result_6 = __disposeResources(env_6);
664
- if (result_6)
665
- await result_6;
666
541
  }
542
+ expect(lines).toMatchSnapshot();
667
543
  });
668
544
  test('sync legacy non-raw data', async () => {
669
- const env_7 = { stack: [], error: void 0, hasError: false };
670
- try {
671
- const f = __addDisposableResource(env_7, await factory(), true);
672
- const syncRules = await updateSyncRules(f, {
673
- content: BASIC_SYNC_RULES
674
- });
675
- const bucketStorage = await f.getInstance(syncRules);
676
- const writer = __addDisposableResource(env_7, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
677
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
678
- await writer.markAllSnapshotDone('0/1');
679
- await writer.save({
680
- sourceTable: testTable,
681
- tag: storage.SaveOperationTag.INSERT,
682
- after: {
683
- id: 't1',
684
- description: 'Test\n"string"',
685
- large_num: 12345678901234567890n
686
- },
687
- afterReplicaId: 't1'
688
- });
689
- await writer.commit('0/1');
690
- const stream = sync.streamResponse({
691
- syncContext,
692
- bucketStorage,
693
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
694
- params: {
695
- buckets: [],
696
- include_checksum: true,
697
- raw_data: false
698
- },
699
- tracker,
700
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
701
- isEncodingAsBson: false
702
- });
703
- const lines = await consumeCheckpointLines(stream);
704
- expect(lines).toMatchSnapshot();
705
- // Specifically check the number - this is the important part of the test
706
- expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
707
- }
708
- catch (e_7) {
709
- env_7.error = e_7;
710
- env_7.hasError = true;
711
- }
712
- finally {
713
- const result_7 = __disposeResources(env_7);
714
- if (result_7)
715
- await result_7;
716
- }
545
+ await using f = await factory();
546
+ const syncRules = await updateSyncRules(f, {
547
+ content: BASIC_SYNC_RULES
548
+ });
549
+ const bucketStorage = await f.getInstance(syncRules);
550
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
551
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
552
+ await writer.markAllSnapshotDone('0/1');
553
+ await writer.save({
554
+ sourceTable: testTable,
555
+ tag: storage.SaveOperationTag.INSERT,
556
+ after: {
557
+ id: 't1',
558
+ description: 'Test\n"string"',
559
+ large_num: 12345678901234567890n
560
+ },
561
+ afterReplicaId: 't1'
562
+ });
563
+ await writer.commit('0/1');
564
+ const stream = sync.streamResponse({
565
+ syncContext,
566
+ bucketStorage,
567
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
568
+ params: {
569
+ buckets: [],
570
+ include_checksum: true,
571
+ raw_data: false
572
+ },
573
+ tracker,
574
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
575
+ isEncodingAsBson: false
576
+ });
577
+ const lines = await consumeCheckpointLines(stream);
578
+ expect(lines).toMatchSnapshot();
579
+ // Specifically check the number - this is the important part of the test
580
+ expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
717
581
  });
718
582
  test('expired token', async () => {
719
- const env_8 = { stack: [], error: void 0, hasError: false };
720
- try {
721
- const f = __addDisposableResource(env_8, await factory(), true);
722
- const syncRules = await updateSyncRules(f, {
723
- content: BASIC_SYNC_RULES
724
- });
725
- const bucketStorage = await f.getInstance(syncRules);
726
- const stream = sync.streamResponse({
727
- syncContext,
728
- bucketStorage,
729
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
730
- params: {
731
- buckets: [],
732
- include_checksum: true,
733
- raw_data: true
734
- },
735
- tracker,
736
- token: new JwtPayload({ sub: '', exp: 0 }),
737
- isEncodingAsBson: false
738
- });
739
- const lines = await consumeCheckpointLines(stream);
740
- expect(lines).toMatchSnapshot();
741
- }
742
- catch (e_8) {
743
- env_8.error = e_8;
744
- env_8.hasError = true;
745
- }
746
- finally {
747
- const result_8 = __disposeResources(env_8);
748
- if (result_8)
749
- await result_8;
750
- }
583
+ await using f = await factory();
584
+ const syncRules = await updateSyncRules(f, {
585
+ content: BASIC_SYNC_RULES
586
+ });
587
+ const bucketStorage = await f.getInstance(syncRules);
588
+ const stream = sync.streamResponse({
589
+ syncContext,
590
+ bucketStorage,
591
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
592
+ params: {
593
+ buckets: [],
594
+ include_checksum: true,
595
+ raw_data: true
596
+ },
597
+ tracker,
598
+ token: new JwtPayload({ sub: '', exp: 0 }),
599
+ isEncodingAsBson: false
600
+ });
601
+ const lines = await consumeCheckpointLines(stream);
602
+ expect(lines).toMatchSnapshot();
751
603
  });
752
604
  test('sync updates to global data', async (context) => {
753
- const env_9 = { stack: [], error: void 0, hasError: false };
754
- try {
755
- const f = __addDisposableResource(env_9, await factory(), true);
756
- const syncRules = await updateSyncRules(f, {
757
- content: BASIC_SYNC_RULES
758
- });
759
- const bucketStorage = await f.getInstance(syncRules);
760
- const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
761
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
762
- // Activate
763
- await writer.markAllSnapshotDone('0/0');
764
- await writer.keepalive('0/0');
765
- const stream = sync.streamResponse({
766
- syncContext,
767
- bucketStorage,
768
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
769
- params: {
770
- buckets: [],
771
- include_checksum: true,
772
- raw_data: true
773
- },
774
- tracker,
775
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
776
- isEncodingAsBson: false
777
- });
778
- const iter = stream[Symbol.asyncIterator]();
779
- context.onTestFinished(() => {
780
- iter.return?.();
781
- });
782
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
783
- await writer.save({
784
- sourceTable: testTable,
785
- tag: storage.SaveOperationTag.INSERT,
786
- after: {
787
- id: 't1',
788
- description: 'Test 1'
789
- },
790
- afterReplicaId: 't1'
791
- });
792
- await writer.commit('0/1');
793
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
794
- await writer.save({
795
- sourceTable: testTable,
796
- tag: storage.SaveOperationTag.INSERT,
797
- after: {
798
- id: 't2',
799
- description: 'Test 2'
800
- },
801
- afterReplicaId: 't2'
802
- });
803
- await writer.commit('0/2');
804
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
805
- }
806
- catch (e_9) {
807
- env_9.error = e_9;
808
- env_9.hasError = true;
809
- }
810
- finally {
811
- const result_9 = __disposeResources(env_9);
812
- if (result_9)
813
- await result_9;
814
- }
605
+ await using f = await factory();
606
+ const syncRules = await updateSyncRules(f, {
607
+ content: BASIC_SYNC_RULES
608
+ });
609
+ const bucketStorage = await f.getInstance(syncRules);
610
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
611
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
612
+ // Activate
613
+ await writer.markAllSnapshotDone('0/0');
614
+ await writer.keepalive('0/0');
615
+ const stream = sync.streamResponse({
616
+ syncContext,
617
+ bucketStorage,
618
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
619
+ params: {
620
+ buckets: [],
621
+ include_checksum: true,
622
+ raw_data: true
623
+ },
624
+ tracker,
625
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
626
+ isEncodingAsBson: false
627
+ });
628
+ const iter = stream[Symbol.asyncIterator]();
629
+ context.onTestFinished(() => {
630
+ iter.return?.();
631
+ });
632
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
633
+ await writer.save({
634
+ sourceTable: testTable,
635
+ tag: storage.SaveOperationTag.INSERT,
636
+ after: {
637
+ id: 't1',
638
+ description: 'Test 1'
639
+ },
640
+ afterReplicaId: 't1'
641
+ });
642
+ await writer.commit('0/1');
643
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
644
+ await writer.save({
645
+ sourceTable: testTable,
646
+ tag: storage.SaveOperationTag.INSERT,
647
+ after: {
648
+ id: 't2',
649
+ description: 'Test 2'
650
+ },
651
+ afterReplicaId: 't2'
652
+ });
653
+ await writer.commit('0/2');
654
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
815
655
  });
816
656
  test('sync updates to parameter query only', async (context) => {
817
- const env_10 = { stack: [], error: void 0, hasError: false };
818
- try {
819
- const f = __addDisposableResource(env_10, await factory(), true);
820
- const syncRules = await updateSyncRules(f, {
821
- content: `bucket_definitions:
657
+ await using f = await factory();
658
+ const syncRules = await updateSyncRules(f, {
659
+ content: `bucket_definitions:
822
660
  by_user:
823
661
  parameters: select users.id as user_id from users where users.id = request.user_id()
824
662
  data:
825
663
  - select * from lists where user_id = bucket.user_id
826
664
  `
827
- });
828
- const bucketStorage = await f.getInstance(syncRules);
829
- const writer = __addDisposableResource(env_10, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
830
- const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
831
- // Activate
832
- await writer.markAllSnapshotDone('0/0');
833
- await writer.keepalive('0/0');
834
- const stream = sync.streamResponse({
835
- syncContext,
836
- bucketStorage,
837
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
838
- params: {
839
- buckets: [],
840
- include_checksum: true,
841
- raw_data: true
842
- },
843
- tracker,
844
- token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
845
- isEncodingAsBson: false
846
- });
847
- const iter = stream[Symbol.asyncIterator]();
848
- context.onTestFinished(() => {
849
- iter.return?.();
850
- });
851
- // Initial empty checkpoint
852
- const checkpoint1 = await getCheckpointLines(iter);
853
- expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([]);
854
- expect(checkpoint1).toMatchSnapshot();
855
- // Add user
856
- await writer.save({
857
- sourceTable: usersTable,
858
- tag: storage.SaveOperationTag.INSERT,
859
- after: {
860
- id: 'user1',
861
- name: 'User 1'
862
- },
863
- afterReplicaId: 'user1'
864
- });
865
- await writer.commit('0/1');
866
- const checkpoint2 = await getCheckpointLines(iter);
867
- const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
868
- expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
869
- expect(checkpoint2).toMatchSnapshot();
870
- }
871
- catch (e_10) {
872
- env_10.error = e_10;
873
- env_10.hasError = true;
874
- }
875
- finally {
876
- const result_10 = __disposeResources(env_10);
877
- if (result_10)
878
- await result_10;
879
- }
665
+ });
666
+ const bucketStorage = await f.getInstance(syncRules);
667
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
668
+ const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
669
+ // Activate
670
+ await writer.markAllSnapshotDone('0/0');
671
+ await writer.keepalive('0/0');
672
+ const stream = sync.streamResponse({
673
+ syncContext,
674
+ bucketStorage,
675
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
676
+ params: {
677
+ buckets: [],
678
+ include_checksum: true,
679
+ raw_data: true
680
+ },
681
+ tracker,
682
+ token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
683
+ isEncodingAsBson: false
684
+ });
685
+ const iter = stream[Symbol.asyncIterator]();
686
+ context.onTestFinished(() => {
687
+ iter.return?.();
688
+ });
689
+ // Initial empty checkpoint
690
+ const checkpoint1 = await getCheckpointLines(iter);
691
+ expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([]);
692
+ expect(checkpoint1).toMatchSnapshot();
693
+ // Add user
694
+ await writer.save({
695
+ sourceTable: usersTable,
696
+ tag: storage.SaveOperationTag.INSERT,
697
+ after: {
698
+ id: 'user1',
699
+ name: 'User 1'
700
+ },
701
+ afterReplicaId: 'user1'
702
+ });
703
+ await writer.commit('0/1');
704
+ const checkpoint2 = await getCheckpointLines(iter);
705
+ const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
706
+ expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
707
+ expect(checkpoint2).toMatchSnapshot();
880
708
  });
881
709
  test('sync updates to data query only', async (context) => {
882
- const env_11 = { stack: [], error: void 0, hasError: false };
883
- try {
884
- const f = __addDisposableResource(env_11, await factory(), true);
885
- const syncRules = await updateSyncRules(f, {
886
- content: `bucket_definitions:
710
+ await using f = await factory();
711
+ const syncRules = await updateSyncRules(f, {
712
+ content: `bucket_definitions:
887
713
  by_user:
888
714
  parameters: select users.id as user_id from users where users.id = request.user_id()
889
715
  data:
890
716
  - select * from lists where user_id = bucket.user_id
891
717
  `
892
- });
893
- const bucketStorage = await f.getInstance(syncRules);
894
- const writer = __addDisposableResource(env_11, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
895
- const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
896
- const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
897
- await writer.markAllSnapshotDone('0/1');
898
- await writer.save({
899
- sourceTable: usersTable,
900
- tag: storage.SaveOperationTag.INSERT,
901
- after: {
902
- id: 'user1',
903
- name: 'User 1'
904
- },
905
- afterReplicaId: 'user1'
906
- });
907
- await writer.commit('0/1');
908
- const stream = sync.streamResponse({
909
- syncContext,
910
- bucketStorage,
911
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
912
- params: {
913
- buckets: [],
914
- include_checksum: true,
915
- raw_data: true
916
- },
917
- tracker,
918
- token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
919
- isEncodingAsBson: false
920
- });
921
- const iter = stream[Symbol.asyncIterator]();
922
- context.onTestFinished(() => {
923
- iter.return?.();
924
- });
925
- const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
926
- const checkpoint1 = await getCheckpointLines(iter);
927
- expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
928
- expect(checkpoint1).toMatchSnapshot();
929
- await writer.save({
930
- sourceTable: listsTable,
931
- tag: storage.SaveOperationTag.INSERT,
932
- after: {
933
- id: 'list1',
934
- user_id: 'user1',
935
- name: 'User 1'
936
- },
937
- afterReplicaId: 'list1'
938
- });
939
- await writer.commit('0/1');
940
- const checkpoint2 = await getCheckpointLines(iter);
941
- expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
942
- expect(checkpoint2).toMatchSnapshot();
943
- }
944
- catch (e_11) {
945
- env_11.error = e_11;
946
- env_11.hasError = true;
947
- }
948
- finally {
949
- const result_11 = __disposeResources(env_11);
950
- if (result_11)
951
- await result_11;
952
- }
718
+ });
719
+ const bucketStorage = await f.getInstance(syncRules);
720
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
721
+ const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
722
+ const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
723
+ await writer.markAllSnapshotDone('0/1');
724
+ await writer.save({
725
+ sourceTable: usersTable,
726
+ tag: storage.SaveOperationTag.INSERT,
727
+ after: {
728
+ id: 'user1',
729
+ name: 'User 1'
730
+ },
731
+ afterReplicaId: 'user1'
732
+ });
733
+ await writer.commit('0/1');
734
+ const stream = sync.streamResponse({
735
+ syncContext,
736
+ bucketStorage,
737
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
738
+ params: {
739
+ buckets: [],
740
+ include_checksum: true,
741
+ raw_data: true
742
+ },
743
+ tracker,
744
+ token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
745
+ isEncodingAsBson: false
746
+ });
747
+ const iter = stream[Symbol.asyncIterator]();
748
+ context.onTestFinished(() => {
749
+ iter.return?.();
750
+ });
751
+ const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
752
+ const checkpoint1 = await getCheckpointLines(iter);
753
+ expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
754
+ expect(checkpoint1).toMatchSnapshot();
755
+ await writer.save({
756
+ sourceTable: listsTable,
757
+ tag: storage.SaveOperationTag.INSERT,
758
+ after: {
759
+ id: 'list1',
760
+ user_id: 'user1',
761
+ name: 'User 1'
762
+ },
763
+ afterReplicaId: 'list1'
764
+ });
765
+ await writer.commit('0/1');
766
+ const checkpoint2 = await getCheckpointLines(iter);
767
+ expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
768
+ expect(checkpoint2).toMatchSnapshot();
953
769
  });
954
770
  test('sync updates to parameter query + data', async (context) => {
955
- const env_12 = { stack: [], error: void 0, hasError: false };
956
- try {
957
- const f = __addDisposableResource(env_12, await factory(), true);
958
- const syncRules = await updateSyncRules(f, {
959
- content: `bucket_definitions:
771
+ await using f = await factory();
772
+ const syncRules = await updateSyncRules(f, {
773
+ content: `bucket_definitions:
960
774
  by_user:
961
775
  parameters: select users.id as user_id from users where users.id = request.user_id()
962
776
  data:
963
777
  - select * from lists where user_id = bucket.user_id
964
778
  `
965
- });
966
- const bucketStorage = await f.getInstance(syncRules);
967
- const writer = __addDisposableResource(env_12, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
968
- const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
969
- const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
970
- // Activate
971
- await writer.markAllSnapshotDone('0/0');
972
- await writer.keepalive('0/0');
973
- const stream = sync.streamResponse({
974
- syncContext,
975
- bucketStorage,
976
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
977
- params: {
978
- buckets: [],
979
- include_checksum: true,
980
- raw_data: true
981
- },
982
- tracker,
983
- token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
984
- isEncodingAsBson: false
985
- });
986
- const iter = stream[Symbol.asyncIterator]();
987
- context.onTestFinished(() => {
988
- iter.return?.();
989
- });
990
- // Initial empty checkpoint
991
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
992
- await writer.markAllSnapshotDone('0/1');
993
- await writer.save({
994
- sourceTable: listsTable,
995
- tag: storage.SaveOperationTag.INSERT,
996
- after: {
997
- id: 'list1',
998
- user_id: 'user1',
999
- name: 'User 1'
1000
- },
1001
- afterReplicaId: 'list1'
1002
- });
1003
- await writer.save({
1004
- sourceTable: usersTable,
1005
- tag: storage.SaveOperationTag.INSERT,
1006
- after: {
1007
- id: 'user1',
1008
- name: 'User 1'
1009
- },
1010
- afterReplicaId: 'user1'
1011
- });
1012
- await writer.commit('0/1');
1013
- const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
1014
- const checkpoint2 = await getCheckpointLines(iter);
1015
- expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
1016
- expect(checkpoint2).toMatchSnapshot();
1017
- }
1018
- catch (e_12) {
1019
- env_12.error = e_12;
1020
- env_12.hasError = true;
1021
- }
1022
- finally {
1023
- const result_12 = __disposeResources(env_12);
1024
- if (result_12)
1025
- await result_12;
1026
- }
779
+ });
780
+ const bucketStorage = await f.getInstance(syncRules);
781
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
782
+ const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
783
+ const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
784
+ // Activate
785
+ await writer.markAllSnapshotDone('0/0');
786
+ await writer.keepalive('0/0');
787
+ const stream = sync.streamResponse({
788
+ syncContext,
789
+ bucketStorage,
790
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
791
+ params: {
792
+ buckets: [],
793
+ include_checksum: true,
794
+ raw_data: true
795
+ },
796
+ tracker,
797
+ token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
798
+ isEncodingAsBson: false
799
+ });
800
+ const iter = stream[Symbol.asyncIterator]();
801
+ context.onTestFinished(() => {
802
+ iter.return?.();
803
+ });
804
+ // Initial empty checkpoint
805
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
806
+ await writer.markAllSnapshotDone('0/1');
807
+ await writer.save({
808
+ sourceTable: listsTable,
809
+ tag: storage.SaveOperationTag.INSERT,
810
+ after: {
811
+ id: 'list1',
812
+ user_id: 'user1',
813
+ name: 'User 1'
814
+ },
815
+ afterReplicaId: 'list1'
816
+ });
817
+ await writer.save({
818
+ sourceTable: usersTable,
819
+ tag: storage.SaveOperationTag.INSERT,
820
+ after: {
821
+ id: 'user1',
822
+ name: 'User 1'
823
+ },
824
+ afterReplicaId: 'user1'
825
+ });
826
+ await writer.commit('0/1');
827
+ const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
828
+ const checkpoint2 = await getCheckpointLines(iter);
829
+ expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
830
+ expect(checkpoint2).toMatchSnapshot();
1027
831
  });
1028
832
  test('expiring token', async (context) => {
1029
- const env_13 = { stack: [], error: void 0, hasError: false };
1030
- try {
1031
- const f = __addDisposableResource(env_13, await factory(), true);
1032
- const syncRules = await updateSyncRules(f, {
1033
- content: BASIC_SYNC_RULES
1034
- });
1035
- const bucketStorage = await f.getInstance(syncRules);
1036
- const writer = __addDisposableResource(env_13, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1037
- // Activate
1038
- await writer.markAllSnapshotDone('0/0');
1039
- await writer.keepalive('0/0');
1040
- const exp = Date.now() / 1000 + 0.1;
1041
- const stream = sync.streamResponse({
1042
- syncContext,
1043
- bucketStorage,
1044
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1045
- params: {
1046
- buckets: [],
1047
- include_checksum: true,
1048
- raw_data: true
1049
- },
1050
- tracker,
1051
- token: new JwtPayload({ sub: '', exp: exp }),
1052
- isEncodingAsBson: false
1053
- });
1054
- const iter = stream[Symbol.asyncIterator]();
1055
- context.onTestFinished(() => {
1056
- iter.return?.();
1057
- });
1058
- const checkpoint = await getCheckpointLines(iter);
1059
- expect(checkpoint).toMatchSnapshot();
1060
- const expLines = await getCheckpointLines(iter);
1061
- expect(expLines).toMatchSnapshot();
1062
- }
1063
- catch (e_13) {
1064
- env_13.error = e_13;
1065
- env_13.hasError = true;
1066
- }
1067
- finally {
1068
- const result_13 = __disposeResources(env_13);
1069
- if (result_13)
1070
- await result_13;
1071
- }
833
+ await using f = await factory();
834
+ const syncRules = await updateSyncRules(f, {
835
+ content: BASIC_SYNC_RULES
836
+ });
837
+ const bucketStorage = await f.getInstance(syncRules);
838
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
839
+ // Activate
840
+ await writer.markAllSnapshotDone('0/0');
841
+ await writer.keepalive('0/0');
842
+ const exp = Date.now() / 1000 + 0.1;
843
+ const stream = sync.streamResponse({
844
+ syncContext,
845
+ bucketStorage,
846
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
847
+ params: {
848
+ buckets: [],
849
+ include_checksum: true,
850
+ raw_data: true
851
+ },
852
+ tracker,
853
+ token: new JwtPayload({ sub: '', exp: exp }),
854
+ isEncodingAsBson: false
855
+ });
856
+ const iter = stream[Symbol.asyncIterator]();
857
+ context.onTestFinished(() => {
858
+ iter.return?.();
859
+ });
860
+ const checkpoint = await getCheckpointLines(iter);
861
+ expect(checkpoint).toMatchSnapshot();
862
+ const expLines = await getCheckpointLines(iter);
863
+ expect(expLines).toMatchSnapshot();
1072
864
  });
1073
865
  test('compacting data - invalidate checkpoint', async (context) => {
1074
- const env_14 = { stack: [], error: void 0, hasError: false };
1075
- try {
1076
- // This tests a case of a compact operation invalidating a checkpoint in the
1077
- // middle of syncing data.
1078
- // This is expected to be rare in practice, but it is important to handle
1079
- // this case correctly to maintain consistency on the client.
1080
- const f = __addDisposableResource(env_14, await factory(), true);
866
+ // This tests a case of a compact operation invalidating a checkpoint in the
867
+ // middle of syncing data.
868
+ // This is expected to be rare in practice, but it is important to handle
869
+ // this case correctly to maintain consistency on the client.
870
+ await using f = await factory();
871
+ const syncRules = await updateSyncRules(f, {
872
+ content: BASIC_SYNC_RULES
873
+ });
874
+ const bucketStorage = await f.getInstance(syncRules);
875
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
876
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
877
+ await writer.markAllSnapshotDone('0/1');
878
+ await writer.save({
879
+ sourceTable: testTable,
880
+ tag: storage.SaveOperationTag.INSERT,
881
+ after: {
882
+ id: 't1',
883
+ description: 'Test 1'
884
+ },
885
+ afterReplicaId: 't1'
886
+ });
887
+ await writer.save({
888
+ sourceTable: testTable,
889
+ tag: storage.SaveOperationTag.INSERT,
890
+ after: {
891
+ id: 't2',
892
+ description: 'Test 2'
893
+ },
894
+ afterReplicaId: 't2'
895
+ });
896
+ await writer.commit('0/1');
897
+ const stream = sync.streamResponse({
898
+ syncContext,
899
+ bucketStorage,
900
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
901
+ params: {
902
+ buckets: [],
903
+ include_checksum: true,
904
+ raw_data: true
905
+ },
906
+ tracker,
907
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
908
+ isEncodingAsBson: false
909
+ });
910
+ const iter = stream[Symbol.asyncIterator]();
911
+ context.onTestFinished(() => {
912
+ iter.return?.();
913
+ });
914
+ // Only consume the first "checkpoint" message, and pause before receiving data.
915
+ const lines = await consumeIterator(iter, { consume: false, isDone: (line) => line?.checkpoint != null });
916
+ expect(lines).toMatchSnapshot();
917
+ expect(lines[0]).toEqual({
918
+ checkpoint: expect.objectContaining({
919
+ last_op_id: '2'
920
+ })
921
+ });
922
+ // Now we save additional data AND compact before continuing.
923
+ // This invalidates the checkpoint we've received above.
924
+ await writer.markAllSnapshotDone('0/1');
925
+ await writer.save({
926
+ sourceTable: testTable,
927
+ tag: storage.SaveOperationTag.UPDATE,
928
+ after: {
929
+ id: 't1',
930
+ description: 'Test 1b'
931
+ },
932
+ afterReplicaId: 't1'
933
+ });
934
+ await writer.save({
935
+ sourceTable: testTable,
936
+ tag: storage.SaveOperationTag.UPDATE,
937
+ after: {
938
+ id: 't2',
939
+ description: 'Test 2b'
940
+ },
941
+ afterReplicaId: 't2'
942
+ });
943
+ await writer.commit('0/2');
944
+ await bucketStorage.compact({
945
+ minBucketChanges: 1,
946
+ minChangeRatio: 0
947
+ });
948
+ const lines2 = await getCheckpointLines(iter, { consume: true });
949
+ // Snapshot test checks for changes in general.
950
+ // The tests after that documents the specific things we're looking for
951
+ // in this test.
952
+ expect(lines2).toMatchSnapshot();
953
+ expect(lines2[0]).toEqual({
954
+ data: expect.objectContaining({
955
+ has_more: false,
956
+ data: [
957
+ // The first two ops have been replaced by a single CLEAR op
958
+ expect.objectContaining({
959
+ op: 'CLEAR'
960
+ })
961
+ ]
962
+ })
963
+ });
964
+ // Note: No checkpoint_complete here, since the checkpoint has been
965
+ // invalidated by the CLEAR op.
966
+ expect(lines2[1]).toEqual({
967
+ checkpoint_diff: expect.objectContaining({
968
+ last_op_id: '4'
969
+ })
970
+ });
971
+ expect(lines2[2]).toEqual({
972
+ data: expect.objectContaining({
973
+ has_more: false,
974
+ data: [
975
+ expect.objectContaining({
976
+ op: 'PUT'
977
+ }),
978
+ expect.objectContaining({
979
+ op: 'PUT'
980
+ })
981
+ ]
982
+ })
983
+ });
984
+ // Now we get a checkpoint_complete
985
+ expect(lines2[3]).toEqual({
986
+ checkpoint_complete: expect.objectContaining({
987
+ last_op_id: '4'
988
+ })
989
+ });
990
+ });
991
+ test('write checkpoint', async () => {
992
+ await using f = await factory();
993
+ const syncRules = await updateSyncRules(f, {
994
+ content: BASIC_SYNC_RULES
995
+ });
996
+ const bucketStorage = f.getInstance(syncRules);
997
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
998
+ await writer.markAllSnapshotDone('0/1');
999
+ // <= the managed write checkpoint LSN below
1000
+ await writer.commit('0/1');
1001
+ const checkpoint = await bucketStorage.createManagedWriteCheckpoint({
1002
+ user_id: 'test',
1003
+ heads: { '1': '1/0' }
1004
+ });
1005
+ const params = {
1006
+ syncContext,
1007
+ bucketStorage,
1008
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1009
+ params: {
1010
+ buckets: [],
1011
+ include_checksum: true,
1012
+ raw_data: true
1013
+ },
1014
+ tracker,
1015
+ token: new JwtPayload({ sub: 'test', exp: Date.now() / 1000 + 10 }),
1016
+ isEncodingAsBson: false
1017
+ };
1018
+ const stream1 = sync.streamResponse(params);
1019
+ const lines1 = await consumeCheckpointLines(stream1);
1020
+ // If write checkpoints are not correctly filtered, this may already
1021
+ // contain the write checkpoint.
1022
+ expect(lines1[0]).toMatchObject({
1023
+ checkpoint: expect.objectContaining({
1024
+ last_op_id: '0',
1025
+ write_checkpoint: undefined
1026
+ })
1027
+ });
1028
+ await writer.markAllSnapshotDone('0/1');
1029
+ // must be >= the managed write checkpoint LSN
1030
+ await writer.commit('1/0');
1031
+ // At this point the LSN has advanced, so the write checkpoint should be
1032
+ // included in the next checkpoint message.
1033
+ const stream2 = sync.streamResponse(params);
1034
+ const lines2 = await consumeCheckpointLines(stream2);
1035
+ expect(lines2[0]).toMatchObject({
1036
+ checkpoint: expect.objectContaining({
1037
+ last_op_id: '0',
1038
+ write_checkpoint: `${checkpoint}`
1039
+ })
1040
+ });
1041
+ });
1042
+ test('encodes sync rules id in buckets for streams', async () => {
1043
+ await using f = await factory();
1044
+ // This test relies making an actual update to sync rules to test the different bucket names.
1045
+ // The actual naming scheme may change, as long as the two buckets have different names.
1046
+ const rules = [
1047
+ `
1048
+ streams:
1049
+ test:
1050
+ auto_subscribe: true
1051
+ query: SELECT * FROM test;
1052
+
1053
+ config:
1054
+ edition: 2
1055
+ `,
1056
+ `
1057
+ streams:
1058
+ test2:
1059
+ auto_subscribe: true
1060
+ query: SELECT * FROM test WHERE 1;
1061
+
1062
+ config:
1063
+ edition: 2
1064
+ `
1065
+ ];
1066
+ for (let i = 0; i < 2; i++) {
1081
1067
  const syncRules = await updateSyncRules(f, {
1082
- content: BASIC_SYNC_RULES
1068
+ content: rules[i]
1083
1069
  });
1084
- const bucketStorage = await f.getInstance(syncRules);
1085
- const writer = __addDisposableResource(env_14, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1086
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1070
+ const bucketStorage = f.getInstance(syncRules);
1071
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1072
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1);
1087
1073
  await writer.markAllSnapshotDone('0/1');
1088
1074
  await writer.save({
1089
1075
  sourceTable: testTable,
@@ -1094,19 +1080,10 @@ bucket_definitions:
1094
1080
  },
1095
1081
  afterReplicaId: 't1'
1096
1082
  });
1097
- await writer.save({
1098
- sourceTable: testTable,
1099
- tag: storage.SaveOperationTag.INSERT,
1100
- after: {
1101
- id: 't2',
1102
- description: 'Test 2'
1103
- },
1104
- afterReplicaId: 't2'
1105
- });
1106
1083
  await writer.commit('0/1');
1107
1084
  const stream = sync.streamResponse({
1108
1085
  syncContext,
1109
- bucketStorage,
1086
+ bucketStorage: bucketStorage,
1110
1087
  syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1111
1088
  params: {
1112
1089
  buckets: [],
@@ -1117,241 +1094,8 @@ bucket_definitions:
1117
1094
  token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
1118
1095
  isEncodingAsBson: false
1119
1096
  });
1120
- const iter = stream[Symbol.asyncIterator]();
1121
- context.onTestFinished(() => {
1122
- iter.return?.();
1123
- });
1124
- // Only consume the first "checkpoint" message, and pause before receiving data.
1125
- const lines = await consumeIterator(iter, { consume: false, isDone: (line) => line?.checkpoint != null });
1097
+ const lines = await consumeCheckpointLines(stream);
1126
1098
  expect(lines).toMatchSnapshot();
1127
- expect(lines[0]).toEqual({
1128
- checkpoint: expect.objectContaining({
1129
- last_op_id: '2'
1130
- })
1131
- });
1132
- // Now we save additional data AND compact before continuing.
1133
- // This invalidates the checkpoint we've received above.
1134
- await writer.markAllSnapshotDone('0/1');
1135
- await writer.save({
1136
- sourceTable: testTable,
1137
- tag: storage.SaveOperationTag.UPDATE,
1138
- after: {
1139
- id: 't1',
1140
- description: 'Test 1b'
1141
- },
1142
- afterReplicaId: 't1'
1143
- });
1144
- await writer.save({
1145
- sourceTable: testTable,
1146
- tag: storage.SaveOperationTag.UPDATE,
1147
- after: {
1148
- id: 't2',
1149
- description: 'Test 2b'
1150
- },
1151
- afterReplicaId: 't2'
1152
- });
1153
- await writer.commit('0/2');
1154
- await bucketStorage.compact({
1155
- minBucketChanges: 1,
1156
- minChangeRatio: 0
1157
- });
1158
- const lines2 = await getCheckpointLines(iter, { consume: true });
1159
- // Snapshot test checks for changes in general.
1160
- // The tests after that documents the specific things we're looking for
1161
- // in this test.
1162
- expect(lines2).toMatchSnapshot();
1163
- expect(lines2[0]).toEqual({
1164
- data: expect.objectContaining({
1165
- has_more: false,
1166
- data: [
1167
- // The first two ops have been replaced by a single CLEAR op
1168
- expect.objectContaining({
1169
- op: 'CLEAR'
1170
- })
1171
- ]
1172
- })
1173
- });
1174
- // Note: No checkpoint_complete here, since the checkpoint has been
1175
- // invalidated by the CLEAR op.
1176
- expect(lines2[1]).toEqual({
1177
- checkpoint_diff: expect.objectContaining({
1178
- last_op_id: '4'
1179
- })
1180
- });
1181
- expect(lines2[2]).toEqual({
1182
- data: expect.objectContaining({
1183
- has_more: false,
1184
- data: [
1185
- expect.objectContaining({
1186
- op: 'PUT'
1187
- }),
1188
- expect.objectContaining({
1189
- op: 'PUT'
1190
- })
1191
- ]
1192
- })
1193
- });
1194
- // Now we get a checkpoint_complete
1195
- expect(lines2[3]).toEqual({
1196
- checkpoint_complete: expect.objectContaining({
1197
- last_op_id: '4'
1198
- })
1199
- });
1200
- }
1201
- catch (e_14) {
1202
- env_14.error = e_14;
1203
- env_14.hasError = true;
1204
- }
1205
- finally {
1206
- const result_14 = __disposeResources(env_14);
1207
- if (result_14)
1208
- await result_14;
1209
- }
1210
- });
1211
- test('write checkpoint', async () => {
1212
- const env_15 = { stack: [], error: void 0, hasError: false };
1213
- try {
1214
- const f = __addDisposableResource(env_15, await factory(), true);
1215
- const syncRules = await updateSyncRules(f, {
1216
- content: BASIC_SYNC_RULES
1217
- });
1218
- const bucketStorage = f.getInstance(syncRules);
1219
- const writer = __addDisposableResource(env_15, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1220
- await writer.markAllSnapshotDone('0/1');
1221
- // <= the managed write checkpoint LSN below
1222
- await writer.commit('0/1');
1223
- const checkpoint = await bucketStorage.createManagedWriteCheckpoint({
1224
- user_id: 'test',
1225
- heads: { '1': '1/0' }
1226
- });
1227
- const params = {
1228
- syncContext,
1229
- bucketStorage,
1230
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1231
- params: {
1232
- buckets: [],
1233
- include_checksum: true,
1234
- raw_data: true
1235
- },
1236
- tracker,
1237
- token: new JwtPayload({ sub: 'test', exp: Date.now() / 1000 + 10 }),
1238
- isEncodingAsBson: false
1239
- };
1240
- const stream1 = sync.streamResponse(params);
1241
- const lines1 = await consumeCheckpointLines(stream1);
1242
- // If write checkpoints are not correctly filtered, this may already
1243
- // contain the write checkpoint.
1244
- expect(lines1[0]).toMatchObject({
1245
- checkpoint: expect.objectContaining({
1246
- last_op_id: '0',
1247
- write_checkpoint: undefined
1248
- })
1249
- });
1250
- await writer.markAllSnapshotDone('0/1');
1251
- // must be >= the managed write checkpoint LSN
1252
- await writer.commit('1/0');
1253
- // At this point the LSN has advanced, so the write checkpoint should be
1254
- // included in the next checkpoint message.
1255
- const stream2 = sync.streamResponse(params);
1256
- const lines2 = await consumeCheckpointLines(stream2);
1257
- expect(lines2[0]).toMatchObject({
1258
- checkpoint: expect.objectContaining({
1259
- last_op_id: '0',
1260
- write_checkpoint: `${checkpoint}`
1261
- })
1262
- });
1263
- }
1264
- catch (e_15) {
1265
- env_15.error = e_15;
1266
- env_15.hasError = true;
1267
- }
1268
- finally {
1269
- const result_15 = __disposeResources(env_15);
1270
- if (result_15)
1271
- await result_15;
1272
- }
1273
- });
1274
- test('encodes sync rules id in buckets for streams', async () => {
1275
- const env_16 = { stack: [], error: void 0, hasError: false };
1276
- try {
1277
- const f = __addDisposableResource(env_16, await factory(), true);
1278
- // This test relies making an actual update to sync rules to test the different bucket names.
1279
- // The actual naming scheme may change, as long as the two buckets have different names.
1280
- const rules = [
1281
- `
1282
- streams:
1283
- test:
1284
- auto_subscribe: true
1285
- query: SELECT * FROM test;
1286
-
1287
- config:
1288
- edition: 2
1289
- `,
1290
- `
1291
- streams:
1292
- test2:
1293
- auto_subscribe: true
1294
- query: SELECT * FROM test WHERE 1;
1295
-
1296
- config:
1297
- edition: 2
1298
- `
1299
- ];
1300
- for (let i = 0; i < 2; i++) {
1301
- const env_17 = { stack: [], error: void 0, hasError: false };
1302
- try {
1303
- const syncRules = await updateSyncRules(f, {
1304
- content: rules[i]
1305
- });
1306
- const bucketStorage = f.getInstance(syncRules);
1307
- const writer = __addDisposableResource(env_17, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1308
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1);
1309
- await writer.markAllSnapshotDone('0/1');
1310
- await writer.save({
1311
- sourceTable: testTable,
1312
- tag: storage.SaveOperationTag.INSERT,
1313
- after: {
1314
- id: 't1',
1315
- description: 'Test 1'
1316
- },
1317
- afterReplicaId: 't1'
1318
- });
1319
- await writer.commit('0/1');
1320
- const stream = sync.streamResponse({
1321
- syncContext,
1322
- bucketStorage: bucketStorage,
1323
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1324
- params: {
1325
- buckets: [],
1326
- include_checksum: true,
1327
- raw_data: true
1328
- },
1329
- tracker,
1330
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
1331
- isEncodingAsBson: false
1332
- });
1333
- const lines = await consumeCheckpointLines(stream);
1334
- expect(lines).toMatchSnapshot();
1335
- }
1336
- catch (e_16) {
1337
- env_17.error = e_16;
1338
- env_17.hasError = true;
1339
- }
1340
- finally {
1341
- const result_16 = __disposeResources(env_17);
1342
- if (result_16)
1343
- await result_16;
1344
- }
1345
- }
1346
- }
1347
- catch (e_17) {
1348
- env_16.error = e_17;
1349
- env_16.hasError = true;
1350
- }
1351
- finally {
1352
- const result_17 = __disposeResources(env_16);
1353
- if (result_17)
1354
- await result_17;
1355
1099
  }
1356
1100
  });
1357
1101
  }