@powersync/service-core-tests 0.15.2 → 0.15.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,55 +1,3 @@
1
- var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
- if (value !== null && value !== void 0) {
3
- if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
- var dispose, inner;
5
- if (async) {
6
- if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
- dispose = value[Symbol.asyncDispose];
8
- }
9
- if (dispose === void 0) {
10
- if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
- dispose = value[Symbol.dispose];
12
- if (async) inner = dispose;
13
- }
14
- if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
- if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
- env.stack.push({ value: value, dispose: dispose, async: async });
17
- }
18
- else if (async) {
19
- env.stack.push({ async: true });
20
- }
21
- return value;
22
- };
23
- var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
- return function (env) {
25
- function fail(e) {
26
- env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
- env.hasError = true;
28
- }
29
- var r, s = 0;
30
- function next() {
31
- while (r = env.stack.pop()) {
32
- try {
33
- if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
- if (r.dispose) {
35
- var result = r.dispose.call(r.value);
36
- if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
- }
38
- else s |= 1;
39
- }
40
- catch (e) {
41
- fail(e);
42
- }
43
- }
44
- if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
- if (env.hasError) throw env.error;
46
- }
47
- return next();
48
- };
49
- })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
- var e = new Error(message);
51
- return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
- });
53
1
  import { createCoreAPIMetrics, JwtPayload, storage, sync, updateSyncRulesFromYaml } from '@powersync/service-core';
54
2
  import { JSONBig } from '@powersync/service-jsonbig';
55
3
  import path from 'path';
@@ -94,67 +42,53 @@ export function registerSyncTests(configOrFactory, options = {}) {
94
42
  }));
95
43
  };
96
44
  test('sync global data', async () => {
97
- const env_1 = { stack: [], error: void 0, hasError: false };
98
- try {
99
- const f = __addDisposableResource(env_1, await factory(), true);
100
- const syncRules = await updateSyncRules(f, {
101
- content: BASIC_SYNC_RULES
102
- });
103
- const bucketStorage = f.getInstance(syncRules);
104
- const writer = __addDisposableResource(env_1, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
105
- const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
106
- await writer.markAllSnapshotDone('0/1');
107
- await writer.save({
108
- sourceTable,
109
- tag: storage.SaveOperationTag.INSERT,
110
- after: {
111
- id: 't1',
112
- description: 'Test 1'
113
- },
114
- afterReplicaId: 't1'
115
- });
116
- await writer.save({
117
- sourceTable,
118
- tag: storage.SaveOperationTag.INSERT,
119
- after: {
120
- id: 't2',
121
- description: 'Test 2'
122
- },
123
- afterReplicaId: 't2'
124
- });
125
- await writer.commit('0/1');
126
- const stream = sync.streamResponse({
127
- syncContext,
128
- bucketStorage: bucketStorage,
129
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
130
- params: {
131
- buckets: [],
132
- include_checksum: true,
133
- raw_data: true
134
- },
135
- tracker,
136
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
137
- isEncodingAsBson: false
138
- });
139
- const lines = await consumeCheckpointLines(stream);
140
- expect(lines).toMatchSnapshot();
141
- }
142
- catch (e_1) {
143
- env_1.error = e_1;
144
- env_1.hasError = true;
145
- }
146
- finally {
147
- const result_1 = __disposeResources(env_1);
148
- if (result_1)
149
- await result_1;
150
- }
45
+ await using f = await factory();
46
+ const syncRules = await updateSyncRules(f, {
47
+ content: BASIC_SYNC_RULES
48
+ });
49
+ const bucketStorage = f.getInstance(syncRules);
50
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
51
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
52
+ await writer.markAllSnapshotDone('0/1');
53
+ await writer.save({
54
+ sourceTable,
55
+ tag: storage.SaveOperationTag.INSERT,
56
+ after: {
57
+ id: 't1',
58
+ description: 'Test 1'
59
+ },
60
+ afterReplicaId: 't1'
61
+ });
62
+ await writer.save({
63
+ sourceTable,
64
+ tag: storage.SaveOperationTag.INSERT,
65
+ after: {
66
+ id: 't2',
67
+ description: 'Test 2'
68
+ },
69
+ afterReplicaId: 't2'
70
+ });
71
+ await writer.commit('0/1');
72
+ const stream = sync.streamResponse({
73
+ syncContext,
74
+ bucketStorage: bucketStorage,
75
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
76
+ params: {
77
+ buckets: [],
78
+ include_checksum: true,
79
+ raw_data: true
80
+ },
81
+ tracker,
82
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
83
+ isEncodingAsBson: false
84
+ });
85
+ const lines = await consumeCheckpointLines(stream);
86
+ expect(lines).toMatchSnapshot();
151
87
  });
152
88
  test('sync buckets in order', async () => {
153
- const env_2 = { stack: [], error: void 0, hasError: false };
154
- try {
155
- const f = __addDisposableResource(env_2, await factory(), true);
156
- const syncRules = await updateSyncRules(f, {
157
- content: `
89
+ await using f = await factory();
90
+ const syncRules = await updateSyncRules(f, {
91
+ content: `
158
92
  bucket_definitions:
159
93
  b0:
160
94
  priority: 2
@@ -165,62 +99,116 @@ bucket_definitions:
165
99
  data:
166
100
  - SELECT * FROM test WHERE LENGTH(id) > 2;
167
101
  `
168
- });
169
- const bucketStorage = f.getInstance(syncRules);
170
- const writer = __addDisposableResource(env_2, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
171
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
172
- await writer.markAllSnapshotDone('0/1');
173
- await writer.save({
174
- sourceTable: testTable,
175
- tag: storage.SaveOperationTag.INSERT,
176
- after: {
177
- id: 't1',
178
- description: 'Test 1'
179
- },
180
- afterReplicaId: 't1'
181
- });
182
- await writer.save({
183
- sourceTable: testTable,
184
- tag: storage.SaveOperationTag.INSERT,
185
- after: {
186
- id: 'earlier',
187
- description: 'Test 2'
188
- },
189
- afterReplicaId: 'earlier'
190
- });
191
- await writer.commit('0/1');
192
- const stream = sync.streamResponse({
193
- syncContext,
194
- bucketStorage,
195
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
196
- params: {
197
- buckets: [],
198
- include_checksum: true,
199
- raw_data: true
200
- },
201
- tracker,
202
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
203
- isEncodingAsBson: false
204
- });
205
- const lines = await consumeCheckpointLines(stream);
206
- expect(lines).toMatchSnapshot();
207
- }
208
- catch (e_2) {
209
- env_2.error = e_2;
210
- env_2.hasError = true;
211
- }
212
- finally {
213
- const result_2 = __disposeResources(env_2);
214
- if (result_2)
215
- await result_2;
216
- }
102
+ });
103
+ const bucketStorage = f.getInstance(syncRules);
104
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
105
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
106
+ await writer.markAllSnapshotDone('0/1');
107
+ await writer.save({
108
+ sourceTable: testTable,
109
+ tag: storage.SaveOperationTag.INSERT,
110
+ after: {
111
+ id: 't1',
112
+ description: 'Test 1'
113
+ },
114
+ afterReplicaId: 't1'
115
+ });
116
+ await writer.save({
117
+ sourceTable: testTable,
118
+ tag: storage.SaveOperationTag.INSERT,
119
+ after: {
120
+ id: 'earlier',
121
+ description: 'Test 2'
122
+ },
123
+ afterReplicaId: 'earlier'
124
+ });
125
+ await writer.commit('0/1');
126
+ const stream = sync.streamResponse({
127
+ syncContext,
128
+ bucketStorage,
129
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
130
+ params: {
131
+ buckets: [],
132
+ include_checksum: true,
133
+ raw_data: true
134
+ },
135
+ tracker,
136
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
137
+ isEncodingAsBson: false
138
+ });
139
+ const lines = await consumeCheckpointLines(stream);
140
+ expect(lines).toMatchSnapshot();
141
+ });
142
+ test('can override priority when subscribing to stream', async () => {
143
+ await using f = await factory();
144
+ const syncRules = await updateSyncRules(f, {
145
+ content: `
146
+ config:
147
+ edition: 3
148
+
149
+ streams:
150
+ todos:
151
+ query: SELECT * FROM test WHERE id IN subscription.parameter('test_ids')
152
+ `
153
+ });
154
+ const bucketStorage = f.getInstance(syncRules);
155
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
156
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
157
+ await writer.markAllSnapshotDone('0/1');
158
+ await writer.save({
159
+ sourceTable: testTable,
160
+ tag: storage.SaveOperationTag.INSERT,
161
+ after: {
162
+ id: 'a',
163
+ description: 'Test 1'
164
+ },
165
+ afterReplicaId: 't1'
166
+ });
167
+ await writer.save({
168
+ sourceTable: testTable,
169
+ tag: storage.SaveOperationTag.INSERT,
170
+ after: {
171
+ id: 'b',
172
+ description: 'Test 2'
173
+ },
174
+ afterReplicaId: 'earlier'
175
+ });
176
+ await writer.commit('0/1');
177
+ const stream = sync.streamResponse({
178
+ syncContext,
179
+ bucketStorage,
180
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
181
+ params: {
182
+ buckets: [],
183
+ include_checksum: true,
184
+ raw_data: true,
185
+ streams: {
186
+ include_defaults: true,
187
+ subscriptions: [
188
+ {
189
+ stream: 'todos',
190
+ parameters: { test_ids: ['a'] },
191
+ override_priority: 0
192
+ },
193
+ {
194
+ stream: 'todos',
195
+ parameters: { test_ids: ['a', 'b'] },
196
+ override_priority: null
197
+ }
198
+ ]
199
+ }
200
+ },
201
+ tracker,
202
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
203
+ isEncodingAsBson: false
204
+ });
205
+ const lines = await consumeCheckpointLines(stream);
206
+ expect(lines).toMatchSnapshot();
217
207
  });
218
208
  test('sync interrupts low-priority buckets on new checkpoints', async () => {
219
- const env_3 = { stack: [], error: void 0, hasError: false };
220
- try {
221
- const f = __addDisposableResource(env_3, await factory(), true);
222
- const syncRules = await updateSyncRules(f, {
223
- content: `
209
+ await using f = await factory();
210
+ const syncRules = await updateSyncRules(f, {
211
+ content: `
224
212
  bucket_definitions:
225
213
  b0:
226
214
  priority: 2
@@ -231,105 +219,93 @@ bucket_definitions:
231
219
  data:
232
220
  - SELECT * FROM test WHERE LENGTH(id) > 5;
233
221
  `
234
- });
235
- const bucketStorage = f.getInstance(syncRules);
236
- const writer = __addDisposableResource(env_3, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
237
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
238
- await writer.markAllSnapshotDone('0/1');
239
- // Initial data: Add one priority row and 10k low-priority rows.
222
+ });
223
+ const bucketStorage = f.getInstance(syncRules);
224
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
225
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
226
+ await writer.markAllSnapshotDone('0/1');
227
+ // Initial data: Add one priority row and 10k low-priority rows.
228
+ await writer.save({
229
+ sourceTable: testTable,
230
+ tag: storage.SaveOperationTag.INSERT,
231
+ after: {
232
+ id: 'highprio',
233
+ description: 'High priority row'
234
+ },
235
+ afterReplicaId: 'highprio'
236
+ });
237
+ for (let i = 0; i < 10_000; i++) {
240
238
  await writer.save({
241
239
  sourceTable: testTable,
242
240
  tag: storage.SaveOperationTag.INSERT,
243
241
  after: {
244
- id: 'highprio',
245
- description: 'High priority row'
242
+ id: `${i}`,
243
+ description: 'low prio'
246
244
  },
247
- afterReplicaId: 'highprio'
245
+ afterReplicaId: `${i}`
248
246
  });
249
- for (let i = 0; i < 10_000; i++) {
250
- await writer.save({
251
- sourceTable: testTable,
252
- tag: storage.SaveOperationTag.INSERT,
253
- after: {
254
- id: `${i}`,
255
- description: 'low prio'
256
- },
257
- afterReplicaId: `${i}`
258
- });
247
+ }
248
+ await writer.commit('0/1');
249
+ const stream = sync.streamResponse({
250
+ syncContext,
251
+ bucketStorage,
252
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
253
+ params: {
254
+ buckets: [],
255
+ include_checksum: true,
256
+ raw_data: true
257
+ },
258
+ tracker,
259
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
260
+ isEncodingAsBson: false
261
+ });
262
+ let sentCheckpoints = 0;
263
+ let sentRows = 0;
264
+ for await (let next of stream) {
265
+ if (typeof next == 'string') {
266
+ next = JSON.parse(next);
259
267
  }
260
- await writer.commit('0/1');
261
- const stream = sync.streamResponse({
262
- syncContext,
263
- bucketStorage,
264
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
265
- params: {
266
- buckets: [],
267
- include_checksum: true,
268
- raw_data: true
269
- },
270
- tracker,
271
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
272
- isEncodingAsBson: false
273
- });
274
- let sentCheckpoints = 0;
275
- let sentRows = 0;
276
- for await (let next of stream) {
277
- if (typeof next == 'string') {
278
- next = JSON.parse(next);
279
- }
280
- if (typeof next === 'object' && next !== null) {
281
- if ('partial_checkpoint_complete' in next) {
282
- if (sentCheckpoints == 1) {
283
- // Save new data to interrupt the low-priority sync.
284
- // Add another high-priority row. This should interrupt the long-running low-priority sync.
285
- await writer.save({
286
- sourceTable: testTable,
287
- tag: storage.SaveOperationTag.INSERT,
288
- after: {
289
- id: 'highprio2',
290
- description: 'Another high-priority row'
291
- },
292
- afterReplicaId: 'highprio2'
293
- });
294
- await writer.commit('0/2');
295
- }
296
- else {
297
- // Low-priority sync from the first checkpoint was interrupted. This should not happen before
298
- // 1000 low-priority items were synchronized.
299
- expect(sentCheckpoints).toBe(2);
300
- expect(sentRows).toBeGreaterThan(1000);
301
- }
268
+ if (typeof next === 'object' && next !== null) {
269
+ if ('partial_checkpoint_complete' in next) {
270
+ if (sentCheckpoints == 1) {
271
+ // Save new data to interrupt the low-priority sync.
272
+ // Add another high-priority row. This should interrupt the long-running low-priority sync.
273
+ await writer.save({
274
+ sourceTable: testTable,
275
+ tag: storage.SaveOperationTag.INSERT,
276
+ after: {
277
+ id: 'highprio2',
278
+ description: 'Another high-priority row'
279
+ },
280
+ afterReplicaId: 'highprio2'
281
+ });
282
+ await writer.commit('0/2');
302
283
  }
303
- if ('checkpoint' in next || 'checkpoint_diff' in next) {
304
- sentCheckpoints += 1;
305
- }
306
- if ('data' in next) {
307
- sentRows += next.data.data.length;
308
- }
309
- if ('checkpoint_complete' in next) {
310
- break;
284
+ else {
285
+ // Low-priority sync from the first checkpoint was interrupted. This should not happen before
286
+ // 1000 low-priority items were synchronized.
287
+ expect(sentCheckpoints).toBe(2);
288
+ expect(sentRows).toBeGreaterThan(1000);
311
289
  }
312
290
  }
291
+ if ('checkpoint' in next || 'checkpoint_diff' in next) {
292
+ sentCheckpoints += 1;
293
+ }
294
+ if ('data' in next) {
295
+ sentRows += next.data.data.length;
296
+ }
297
+ if ('checkpoint_complete' in next) {
298
+ break;
299
+ }
313
300
  }
314
- expect(sentCheckpoints).toBe(2);
315
- expect(sentRows).toBe(10002);
316
- }
317
- catch (e_3) {
318
- env_3.error = e_3;
319
- env_3.hasError = true;
320
- }
321
- finally {
322
- const result_3 = __disposeResources(env_3);
323
- if (result_3)
324
- await result_3;
325
301
  }
302
+ expect(sentCheckpoints).toBe(2);
303
+ expect(sentRows).toBe(10002);
326
304
  });
327
305
  test('sync interruptions with unrelated data', async () => {
328
- const env_4 = { stack: [], error: void 0, hasError: false };
329
- try {
330
- const f = __addDisposableResource(env_4, await factory(), true);
331
- const syncRules = await updateSyncRules(f, {
332
- content: `
306
+ await using f = await factory();
307
+ const syncRules = await updateSyncRules(f, {
308
+ content: `
333
309
  bucket_definitions:
334
310
  b0:
335
311
  priority: 2
@@ -341,128 +317,116 @@ bucket_definitions:
341
317
  data:
342
318
  - SELECT * FROM test WHERE LENGTH(id) > 5 AND description = bucket.user_id;
343
319
  `
344
- });
345
- const bucketStorage = f.getInstance(syncRules);
346
- const writer = __addDisposableResource(env_4, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
347
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
348
- await writer.markAllSnapshotDone('0/1');
349
- // Initial data: Add one priority row and 10k low-priority rows.
320
+ });
321
+ const bucketStorage = f.getInstance(syncRules);
322
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
323
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
324
+ await writer.markAllSnapshotDone('0/1');
325
+ // Initial data: Add one priority row and 10k low-priority rows.
326
+ await writer.save({
327
+ sourceTable: testTable,
328
+ tag: storage.SaveOperationTag.INSERT,
329
+ after: {
330
+ id: 'highprio',
331
+ description: 'user_one'
332
+ },
333
+ afterReplicaId: 'highprio'
334
+ });
335
+ for (let i = 0; i < 10_000; i++) {
350
336
  await writer.save({
351
337
  sourceTable: testTable,
352
338
  tag: storage.SaveOperationTag.INSERT,
353
339
  after: {
354
- id: 'highprio',
355
- description: 'user_one'
340
+ id: `${i}`,
341
+ description: 'low prio'
356
342
  },
357
- afterReplicaId: 'highprio'
343
+ afterReplicaId: `${i}`
358
344
  });
359
- for (let i = 0; i < 10_000; i++) {
360
- await writer.save({
361
- sourceTable: testTable,
362
- tag: storage.SaveOperationTag.INSERT,
363
- after: {
364
- id: `${i}`,
365
- description: 'low prio'
366
- },
367
- afterReplicaId: `${i}`
368
- });
345
+ }
346
+ await writer.commit('0/1');
347
+ const stream = sync.streamResponse({
348
+ syncContext,
349
+ bucketStorage,
350
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
351
+ params: {
352
+ buckets: [],
353
+ include_checksum: true,
354
+ raw_data: true
355
+ },
356
+ tracker,
357
+ token: new JwtPayload({ sub: 'user_one', exp: Date.now() / 1000 + 100000 }),
358
+ isEncodingAsBson: false
359
+ });
360
+ let sentCheckpoints = 0;
361
+ let completedCheckpoints = 0;
362
+ let sentRows = 0;
363
+ // Expected flow:
364
+ // 1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
365
+ // 2. We insert a new row that is not part of a bucket relevant to this stream.
366
+ // 3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
367
+ // 4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
368
+ for await (let next of stream) {
369
+ if (typeof next == 'string') {
370
+ next = JSON.parse(next);
369
371
  }
370
- await writer.commit('0/1');
371
- const stream = sync.streamResponse({
372
- syncContext,
373
- bucketStorage,
374
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
375
- params: {
376
- buckets: [],
377
- include_checksum: true,
378
- raw_data: true
379
- },
380
- tracker,
381
- token: new JwtPayload({ sub: 'user_one', exp: Date.now() / 1000 + 100000 }),
382
- isEncodingAsBson: false
383
- });
384
- let sentCheckpoints = 0;
385
- let completedCheckpoints = 0;
386
- let sentRows = 0;
387
- // Expected flow:
388
- // 1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
389
- // 2. We insert a new row that is not part of a bucket relevant to this stream.
390
- // 3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
391
- // 4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
392
- for await (let next of stream) {
393
- if (typeof next == 'string') {
394
- next = JSON.parse(next);
395
- }
396
- if (typeof next === 'object' && next !== null) {
397
- if ('partial_checkpoint_complete' in next) {
398
- if (sentCheckpoints == 1) {
399
- // Add a high-priority row that doesn't affect this sync stream.
400
- await writer.save({
401
- sourceTable: testTable,
402
- tag: storage.SaveOperationTag.INSERT,
403
- after: {
404
- id: 'highprio2',
405
- description: 'user_two'
406
- },
407
- afterReplicaId: 'highprio2'
408
- });
409
- await writer.commit('0/2');
410
- }
411
- else {
412
- expect(sentCheckpoints).toBe(2);
413
- expect(sentRows).toBe(10002);
414
- }
372
+ if (typeof next === 'object' && next !== null) {
373
+ if ('partial_checkpoint_complete' in next) {
374
+ if (sentCheckpoints == 1) {
375
+ // Add a high-priority row that doesn't affect this sync stream.
376
+ await writer.save({
377
+ sourceTable: testTable,
378
+ tag: storage.SaveOperationTag.INSERT,
379
+ after: {
380
+ id: 'highprio2',
381
+ description: 'user_two'
382
+ },
383
+ afterReplicaId: 'highprio2'
384
+ });
385
+ await writer.commit('0/2');
415
386
  }
416
- if ('checkpoint' in next || 'checkpoint_diff' in next) {
417
- sentCheckpoints += 1;
387
+ else {
388
+ expect(sentCheckpoints).toBe(2);
389
+ expect(sentRows).toBe(10002);
418
390
  }
419
- if ('data' in next) {
420
- sentRows += next.data.data.length;
391
+ }
392
+ if ('checkpoint' in next || 'checkpoint_diff' in next) {
393
+ sentCheckpoints += 1;
394
+ }
395
+ if ('data' in next) {
396
+ sentRows += next.data.data.length;
397
+ }
398
+ if ('checkpoint_complete' in next) {
399
+ completedCheckpoints++;
400
+ if (completedCheckpoints == 2) {
401
+ break;
421
402
  }
422
- if ('checkpoint_complete' in next) {
423
- completedCheckpoints++;
424
- if (completedCheckpoints == 2) {
425
- break;
426
- }
427
- if (completedCheckpoints == 1) {
428
- expect(sentRows).toBe(10001);
429
- // Add a high-priority row that affects this sync stream.
430
- await writer.save({
431
- sourceTable: testTable,
432
- tag: storage.SaveOperationTag.INSERT,
433
- after: {
434
- id: 'highprio3',
435
- description: 'user_one'
436
- },
437
- afterReplicaId: 'highprio3'
438
- });
439
- await writer.commit('0/3');
440
- }
403
+ if (completedCheckpoints == 1) {
404
+ expect(sentRows).toBe(10001);
405
+ // Add a high-priority row that affects this sync stream.
406
+ await writer.save({
407
+ sourceTable: testTable,
408
+ tag: storage.SaveOperationTag.INSERT,
409
+ after: {
410
+ id: 'highprio3',
411
+ description: 'user_one'
412
+ },
413
+ afterReplicaId: 'highprio3'
414
+ });
415
+ await writer.commit('0/3');
441
416
  }
442
417
  }
443
418
  }
444
- expect(sentCheckpoints).toBe(2);
445
- expect(sentRows).toBe(10002);
446
- }
447
- catch (e_4) {
448
- env_4.error = e_4;
449
- env_4.hasError = true;
450
- }
451
- finally {
452
- const result_4 = __disposeResources(env_4);
453
- if (result_4)
454
- await result_4;
455
419
  }
420
+ expect(sentCheckpoints).toBe(2);
421
+ expect(sentRows).toBe(10002);
456
422
  });
457
423
  test('sync interrupts low-priority buckets on new checkpoints (2)', async () => {
458
- const env_5 = { stack: [], error: void 0, hasError: false };
459
- try {
460
- const f = __addDisposableResource(env_5, await factory(), true);
461
- // bucket0a -> send all data
462
- // then interrupt checkpoint with new data for all buckets
463
- // -> data for all buckets should be sent in the new checkpoint
464
- const syncRules = await updateSyncRules(f, {
465
- content: `
424
+ await using f = await factory();
425
+ // bucket0a -> send all data
426
+ // then interrupt checkpoint with new data for all buckets
427
+ // -> data for all buckets should be sent in the new checkpoint
428
+ const syncRules = await updateSyncRules(f, {
429
+ content: `
466
430
  bucket_definitions:
467
431
  b0a:
468
432
  priority: 2
@@ -477,613 +441,701 @@ bucket_definitions:
477
441
  data:
478
442
  - SELECT * FROM test WHERE LENGTH(id) > 5;
479
443
  `
480
- });
481
- const bucketStorage = f.getInstance(syncRules);
482
- const writer = __addDisposableResource(env_5, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
483
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
484
- await writer.markAllSnapshotDone('0/1');
485
- // Initial data: Add one priority row and 10k low-priority rows.
444
+ });
445
+ const bucketStorage = f.getInstance(syncRules);
446
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
447
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
448
+ await writer.markAllSnapshotDone('0/1');
449
+ // Initial data: Add one priority row and 10k low-priority rows.
450
+ await writer.save({
451
+ sourceTable: testTable,
452
+ tag: storage.SaveOperationTag.INSERT,
453
+ after: {
454
+ id: 'highprio',
455
+ description: 'High priority row'
456
+ },
457
+ afterReplicaId: 'highprio'
458
+ });
459
+ for (let i = 0; i < 2_000; i++) {
486
460
  await writer.save({
487
461
  sourceTable: testTable,
488
462
  tag: storage.SaveOperationTag.INSERT,
489
463
  after: {
490
- id: 'highprio',
491
- description: 'High priority row'
464
+ id: `${i}`,
465
+ description: 'low prio'
492
466
  },
493
- afterReplicaId: 'highprio'
467
+ afterReplicaId: `${i}`
494
468
  });
495
- for (let i = 0; i < 2_000; i++) {
496
- await writer.save({
497
- sourceTable: testTable,
498
- tag: storage.SaveOperationTag.INSERT,
499
- after: {
500
- id: `${i}`,
501
- description: 'low prio'
502
- },
503
- afterReplicaId: `${i}`
504
- });
469
+ }
470
+ await writer.commit('0/1');
471
+ const stream = sync.streamResponse({
472
+ syncContext,
473
+ bucketStorage,
474
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
475
+ params: {
476
+ buckets: [],
477
+ include_checksum: true,
478
+ raw_data: true
479
+ },
480
+ tracker,
481
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
482
+ isEncodingAsBson: false
483
+ });
484
+ let sentRows = 0;
485
+ let lines = [];
486
+ for await (let next of stream) {
487
+ if (typeof next == 'string') {
488
+ next = JSON.parse(next);
505
489
  }
506
- await writer.commit('0/1');
507
- const stream = sync.streamResponse({
508
- syncContext,
509
- bucketStorage,
510
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
511
- params: {
512
- buckets: [],
513
- include_checksum: true,
514
- raw_data: true
515
- },
516
- tracker,
517
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
518
- isEncodingAsBson: false
519
- });
520
- let sentRows = 0;
521
- let lines = [];
522
- for await (let next of stream) {
523
- if (typeof next == 'string') {
524
- next = JSON.parse(next);
490
+ if (typeof next === 'object' && next !== null) {
491
+ if ('partial_checkpoint_complete' in next) {
492
+ lines.push(next);
525
493
  }
526
- if (typeof next === 'object' && next !== null) {
527
- if ('partial_checkpoint_complete' in next) {
528
- lines.push(next);
529
- }
530
- if ('checkpoint' in next || 'checkpoint_diff' in next) {
531
- lines.push(next);
494
+ if ('checkpoint' in next || 'checkpoint_diff' in next) {
495
+ lines.push(next);
496
+ }
497
+ if ('data' in next) {
498
+ lines.push({ data: { ...next.data, data: undefined } });
499
+ sentRows += next.data.data.length;
500
+ if (sentRows == 1001) {
501
+ // Save new data to interrupt the low-priority sync.
502
+ // Add another high-priority row. This should interrupt the long-running low-priority sync.
503
+ await writer.save({
504
+ sourceTable: testTable,
505
+ tag: storage.SaveOperationTag.INSERT,
506
+ after: {
507
+ id: 'highprio2',
508
+ description: 'Another high-priority row'
509
+ },
510
+ afterReplicaId: 'highprio2'
511
+ });
512
+ // Also add a low-priority row
513
+ await writer.save({
514
+ sourceTable: testTable,
515
+ tag: storage.SaveOperationTag.INSERT,
516
+ after: {
517
+ id: '2001',
518
+ description: 'Another low-priority row'
519
+ },
520
+ afterReplicaId: '2001'
521
+ });
522
+ await writer.commit('0/2');
532
523
  }
533
- if ('data' in next) {
534
- lines.push({ data: { ...next.data, data: undefined } });
535
- sentRows += next.data.data.length;
536
- if (sentRows == 1001) {
537
- // Save new data to interrupt the low-priority sync.
538
- // Add another high-priority row. This should interrupt the long-running low-priority sync.
539
- await writer.save({
540
- sourceTable: testTable,
541
- tag: storage.SaveOperationTag.INSERT,
542
- after: {
543
- id: 'highprio2',
544
- description: 'Another high-priority row'
545
- },
546
- afterReplicaId: 'highprio2'
547
- });
548
- // Also add a low-priority row
549
- await writer.save({
550
- sourceTable: testTable,
551
- tag: storage.SaveOperationTag.INSERT,
552
- after: {
553
- id: '2001',
554
- description: 'Another low-priority row'
555
- },
556
- afterReplicaId: '2001'
557
- });
558
- await writer.commit('0/2');
559
- }
560
- if (sentRows >= 1000 && sentRows <= 2001) {
561
- // pause for a bit to give the stream time to process interruptions.
562
- // This covers the data batch above and the next one.
563
- await timers.setTimeout(50);
564
- }
565
- }
566
- if ('checkpoint_complete' in next) {
567
- lines.push(next);
568
- break;
524
+ if (sentRows >= 1000 && sentRows <= 2001) {
525
+ // pause for a bit to give the stream time to process interruptions.
526
+ // This covers the data batch above and the next one.
527
+ await timers.setTimeout(50);
569
528
  }
570
529
  }
530
+ if ('checkpoint_complete' in next) {
531
+ lines.push(next);
532
+ break;
533
+ }
571
534
  }
572
- // Expected lines (full details in snapshot):
573
- //
574
- // checkpoint (4001)
575
- // data (b1[] 0 -> 1)
576
- // partial_checkpoint_complete (4001, priority 1)
577
- // data (b0a[], 0 -> 2000)
578
- // ## adds new data, interrupting the checkpoint
579
- // data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
580
- // checkpoint_diff (4004)
581
- // data (b1[], 1 -> 4002)
582
- // partial_checkpoint_complete (4004, priority 1)
583
- // data (b0a[], 4000 -> 4003)
584
- // data (b0b[], 0 -> 1999)
585
- // data (b0b[], 1999 -> 3999)
586
- // data (b0b[], 3999 -> 4004)
587
- // checkpoint_complete (4004)
588
- expect(lines).toMatchSnapshot();
589
- expect(sentRows).toBe(4004);
590
- }
591
- catch (e_5) {
592
- env_5.error = e_5;
593
- env_5.hasError = true;
594
- }
595
- finally {
596
- const result_5 = __disposeResources(env_5);
597
- if (result_5)
598
- await result_5;
599
535
  }
536
+ // Expected lines (full details in snapshot):
537
+ //
538
+ // checkpoint (4001)
539
+ // data (b1[] 0 -> 1)
540
+ // partial_checkpoint_complete (4001, priority 1)
541
+ // data (b0a[], 0 -> 2000)
542
+ // ## adds new data, interrupting the checkpoint
543
+ // data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
544
+ // checkpoint_diff (4004)
545
+ // data (b1[], 1 -> 4002)
546
+ // partial_checkpoint_complete (4004, priority 1)
547
+ // data (b0a[], 4000 -> 4003)
548
+ // data (b0b[], 0 -> 1999)
549
+ // data (b0b[], 1999 -> 3999)
550
+ // data (b0b[], 3999 -> 4004)
551
+ // checkpoint_complete (4004)
552
+ expect(lines).toMatchSnapshot();
553
+ expect(sentRows).toBe(4004);
600
554
  });
601
555
  test('sends checkpoint complete line for empty checkpoint', async () => {
602
- const env_6 = { stack: [], error: void 0, hasError: false };
603
- try {
604
- const f = __addDisposableResource(env_6, await factory(), true);
605
- const syncRules = await updateSyncRules(f, {
606
- content: BASIC_SYNC_RULES
607
- });
608
- const bucketStorage = f.getInstance(syncRules);
609
- const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
610
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
611
- await writer.markAllSnapshotDone('0/1');
612
- await writer.save({
613
- sourceTable: testTable,
614
- tag: storage.SaveOperationTag.INSERT,
615
- after: {
616
- id: 't1',
617
- description: 'sync'
618
- },
619
- afterReplicaId: 't1'
620
- });
621
- await writer.commit('0/1');
622
- const stream = sync.streamResponse({
623
- syncContext,
624
- bucketStorage,
625
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
626
- params: {
627
- buckets: [],
628
- include_checksum: true,
629
- raw_data: true
630
- },
631
- tracker,
632
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 100000 }),
633
- isEncodingAsBson: false
634
- });
635
- const lines = [];
636
- let receivedCompletions = 0;
637
- for await (let next of stream) {
638
- if (typeof next == 'string') {
639
- next = JSON.parse(next);
640
- }
641
- lines.push(next);
642
- if (typeof next === 'object' && next !== null) {
643
- if ('checkpoint_complete' in next) {
644
- receivedCompletions++;
645
- if (receivedCompletions == 1) {
646
- // Trigger an empty bucket update.
647
- await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } });
648
- await writer.commit('1/0');
649
- }
650
- else {
651
- break;
652
- }
556
+ await using f = await factory();
557
+ const syncRules = await updateSyncRules(f, {
558
+ content: BASIC_SYNC_RULES
559
+ });
560
+ const bucketStorage = f.getInstance(syncRules);
561
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
562
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
563
+ await writer.markAllSnapshotDone('0/1');
564
+ await writer.save({
565
+ sourceTable: testTable,
566
+ tag: storage.SaveOperationTag.INSERT,
567
+ after: {
568
+ id: 't1',
569
+ description: 'sync'
570
+ },
571
+ afterReplicaId: 't1'
572
+ });
573
+ await writer.commit('0/1');
574
+ const stream = sync.streamResponse({
575
+ syncContext,
576
+ bucketStorage,
577
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
578
+ params: {
579
+ buckets: [],
580
+ include_checksum: true,
581
+ raw_data: true
582
+ },
583
+ tracker,
584
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 100000 }),
585
+ isEncodingAsBson: false
586
+ });
587
+ const lines = [];
588
+ let receivedCompletions = 0;
589
+ for await (let next of stream) {
590
+ if (typeof next == 'string') {
591
+ next = JSON.parse(next);
592
+ }
593
+ lines.push(next);
594
+ if (typeof next === 'object' && next !== null) {
595
+ if ('checkpoint_complete' in next) {
596
+ receivedCompletions++;
597
+ if (receivedCompletions == 1) {
598
+ // Trigger an empty bucket update.
599
+ await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } });
600
+ await writer.commit('1/0');
601
+ }
602
+ else {
603
+ break;
653
604
  }
654
605
  }
655
606
  }
656
- expect(lines).toMatchSnapshot();
657
- }
658
- catch (e_6) {
659
- env_6.error = e_6;
660
- env_6.hasError = true;
661
- }
662
- finally {
663
- const result_6 = __disposeResources(env_6);
664
- if (result_6)
665
- await result_6;
666
607
  }
608
+ expect(lines).toMatchSnapshot();
667
609
  });
668
610
  test('sync legacy non-raw data', async () => {
669
- const env_7 = { stack: [], error: void 0, hasError: false };
670
- try {
671
- const f = __addDisposableResource(env_7, await factory(), true);
672
- const syncRules = await updateSyncRules(f, {
673
- content: BASIC_SYNC_RULES
674
- });
675
- const bucketStorage = await f.getInstance(syncRules);
676
- const writer = __addDisposableResource(env_7, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
677
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
678
- await writer.markAllSnapshotDone('0/1');
679
- await writer.save({
680
- sourceTable: testTable,
681
- tag: storage.SaveOperationTag.INSERT,
682
- after: {
683
- id: 't1',
684
- description: 'Test\n"string"',
685
- large_num: 12345678901234567890n
686
- },
687
- afterReplicaId: 't1'
688
- });
689
- await writer.commit('0/1');
690
- const stream = sync.streamResponse({
691
- syncContext,
692
- bucketStorage,
693
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
694
- params: {
695
- buckets: [],
696
- include_checksum: true,
697
- raw_data: false
698
- },
699
- tracker,
700
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
701
- isEncodingAsBson: false
702
- });
703
- const lines = await consumeCheckpointLines(stream);
704
- expect(lines).toMatchSnapshot();
705
- // Specifically check the number - this is the important part of the test
706
- expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
707
- }
708
- catch (e_7) {
709
- env_7.error = e_7;
710
- env_7.hasError = true;
711
- }
712
- finally {
713
- const result_7 = __disposeResources(env_7);
714
- if (result_7)
715
- await result_7;
716
- }
611
+ await using f = await factory();
612
+ const syncRules = await updateSyncRules(f, {
613
+ content: BASIC_SYNC_RULES
614
+ });
615
+ const bucketStorage = await f.getInstance(syncRules);
616
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
617
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
618
+ await writer.markAllSnapshotDone('0/1');
619
+ await writer.save({
620
+ sourceTable: testTable,
621
+ tag: storage.SaveOperationTag.INSERT,
622
+ after: {
623
+ id: 't1',
624
+ description: 'Test\n"string"',
625
+ large_num: 12345678901234567890n
626
+ },
627
+ afterReplicaId: 't1'
628
+ });
629
+ await writer.commit('0/1');
630
+ const stream = sync.streamResponse({
631
+ syncContext,
632
+ bucketStorage,
633
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
634
+ params: {
635
+ buckets: [],
636
+ include_checksum: true,
637
+ raw_data: false
638
+ },
639
+ tracker,
640
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
641
+ isEncodingAsBson: false
642
+ });
643
+ const lines = await consumeCheckpointLines(stream);
644
+ expect(lines).toMatchSnapshot();
645
+ // Specifically check the number - this is the important part of the test
646
+ expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
717
647
  });
718
648
  test('expired token', async () => {
719
- const env_8 = { stack: [], error: void 0, hasError: false };
720
- try {
721
- const f = __addDisposableResource(env_8, await factory(), true);
722
- const syncRules = await updateSyncRules(f, {
723
- content: BASIC_SYNC_RULES
724
- });
725
- const bucketStorage = await f.getInstance(syncRules);
726
- const stream = sync.streamResponse({
727
- syncContext,
728
- bucketStorage,
729
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
730
- params: {
731
- buckets: [],
732
- include_checksum: true,
733
- raw_data: true
734
- },
735
- tracker,
736
- token: new JwtPayload({ sub: '', exp: 0 }),
737
- isEncodingAsBson: false
738
- });
739
- const lines = await consumeCheckpointLines(stream);
740
- expect(lines).toMatchSnapshot();
741
- }
742
- catch (e_8) {
743
- env_8.error = e_8;
744
- env_8.hasError = true;
745
- }
746
- finally {
747
- const result_8 = __disposeResources(env_8);
748
- if (result_8)
749
- await result_8;
750
- }
649
+ await using f = await factory();
650
+ const syncRules = await updateSyncRules(f, {
651
+ content: BASIC_SYNC_RULES
652
+ });
653
+ const bucketStorage = await f.getInstance(syncRules);
654
+ const stream = sync.streamResponse({
655
+ syncContext,
656
+ bucketStorage,
657
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
658
+ params: {
659
+ buckets: [],
660
+ include_checksum: true,
661
+ raw_data: true
662
+ },
663
+ tracker,
664
+ token: new JwtPayload({ sub: '', exp: 0 }),
665
+ isEncodingAsBson: false
666
+ });
667
+ const lines = await consumeCheckpointLines(stream);
668
+ expect(lines).toMatchSnapshot();
751
669
  });
752
670
  test('sync updates to global data', async (context) => {
753
- const env_9 = { stack: [], error: void 0, hasError: false };
754
- try {
755
- const f = __addDisposableResource(env_9, await factory(), true);
756
- const syncRules = await updateSyncRules(f, {
757
- content: BASIC_SYNC_RULES
758
- });
759
- const bucketStorage = await f.getInstance(syncRules);
760
- const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
761
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
762
- // Activate
763
- await writer.markAllSnapshotDone('0/0');
764
- await writer.keepalive('0/0');
765
- const stream = sync.streamResponse({
766
- syncContext,
767
- bucketStorage,
768
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
769
- params: {
770
- buckets: [],
771
- include_checksum: true,
772
- raw_data: true
773
- },
774
- tracker,
775
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
776
- isEncodingAsBson: false
777
- });
778
- const iter = stream[Symbol.asyncIterator]();
779
- context.onTestFinished(() => {
780
- iter.return?.();
781
- });
782
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
783
- await writer.save({
784
- sourceTable: testTable,
785
- tag: storage.SaveOperationTag.INSERT,
786
- after: {
787
- id: 't1',
788
- description: 'Test 1'
789
- },
790
- afterReplicaId: 't1'
791
- });
792
- await writer.commit('0/1');
793
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
794
- await writer.save({
795
- sourceTable: testTable,
796
- tag: storage.SaveOperationTag.INSERT,
797
- after: {
798
- id: 't2',
799
- description: 'Test 2'
800
- },
801
- afterReplicaId: 't2'
802
- });
803
- await writer.commit('0/2');
804
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
805
- }
806
- catch (e_9) {
807
- env_9.error = e_9;
808
- env_9.hasError = true;
809
- }
810
- finally {
811
- const result_9 = __disposeResources(env_9);
812
- if (result_9)
813
- await result_9;
814
- }
671
+ await using f = await factory();
672
+ const syncRules = await updateSyncRules(f, {
673
+ content: BASIC_SYNC_RULES
674
+ });
675
+ const bucketStorage = await f.getInstance(syncRules);
676
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
677
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
678
+ // Activate
679
+ await writer.markAllSnapshotDone('0/0');
680
+ await writer.keepalive('0/0');
681
+ const stream = sync.streamResponse({
682
+ syncContext,
683
+ bucketStorage,
684
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
685
+ params: {
686
+ buckets: [],
687
+ include_checksum: true,
688
+ raw_data: true
689
+ },
690
+ tracker,
691
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
692
+ isEncodingAsBson: false
693
+ });
694
+ const iter = stream[Symbol.asyncIterator]();
695
+ context.onTestFinished(() => {
696
+ iter.return?.();
697
+ });
698
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
699
+ await writer.save({
700
+ sourceTable: testTable,
701
+ tag: storage.SaveOperationTag.INSERT,
702
+ after: {
703
+ id: 't1',
704
+ description: 'Test 1'
705
+ },
706
+ afterReplicaId: 't1'
707
+ });
708
+ await writer.commit('0/1');
709
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
710
+ await writer.save({
711
+ sourceTable: testTable,
712
+ tag: storage.SaveOperationTag.INSERT,
713
+ after: {
714
+ id: 't2',
715
+ description: 'Test 2'
716
+ },
717
+ afterReplicaId: 't2'
718
+ });
719
+ await writer.commit('0/2');
720
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
815
721
  });
816
722
  test('sync updates to parameter query only', async (context) => {
817
- const env_10 = { stack: [], error: void 0, hasError: false };
818
- try {
819
- const f = __addDisposableResource(env_10, await factory(), true);
820
- const syncRules = await updateSyncRules(f, {
821
- content: `bucket_definitions:
723
+ await using f = await factory();
724
+ const syncRules = await updateSyncRules(f, {
725
+ content: `bucket_definitions:
822
726
  by_user:
823
727
  parameters: select users.id as user_id from users where users.id = request.user_id()
824
728
  data:
825
729
  - select * from lists where user_id = bucket.user_id
826
730
  `
827
- });
828
- const bucketStorage = await f.getInstance(syncRules);
829
- const writer = __addDisposableResource(env_10, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
830
- const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
831
- // Activate
832
- await writer.markAllSnapshotDone('0/0');
833
- await writer.keepalive('0/0');
834
- const stream = sync.streamResponse({
835
- syncContext,
836
- bucketStorage,
837
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
838
- params: {
839
- buckets: [],
840
- include_checksum: true,
841
- raw_data: true
842
- },
843
- tracker,
844
- token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
845
- isEncodingAsBson: false
846
- });
847
- const iter = stream[Symbol.asyncIterator]();
848
- context.onTestFinished(() => {
849
- iter.return?.();
850
- });
851
- // Initial empty checkpoint
852
- const checkpoint1 = await getCheckpointLines(iter);
853
- expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([]);
854
- expect(checkpoint1).toMatchSnapshot();
855
- // Add user
856
- await writer.save({
857
- sourceTable: usersTable,
858
- tag: storage.SaveOperationTag.INSERT,
859
- after: {
860
- id: 'user1',
861
- name: 'User 1'
862
- },
863
- afterReplicaId: 'user1'
864
- });
865
- await writer.commit('0/1');
866
- const checkpoint2 = await getCheckpointLines(iter);
867
- const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
868
- expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
869
- expect(checkpoint2).toMatchSnapshot();
870
- }
871
- catch (e_10) {
872
- env_10.error = e_10;
873
- env_10.hasError = true;
874
- }
875
- finally {
876
- const result_10 = __disposeResources(env_10);
877
- if (result_10)
878
- await result_10;
879
- }
731
+ });
732
+ const bucketStorage = await f.getInstance(syncRules);
733
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
734
+ const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
735
+ // Activate
736
+ await writer.markAllSnapshotDone('0/0');
737
+ await writer.keepalive('0/0');
738
+ const stream = sync.streamResponse({
739
+ syncContext,
740
+ bucketStorage,
741
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
742
+ params: {
743
+ buckets: [],
744
+ include_checksum: true,
745
+ raw_data: true
746
+ },
747
+ tracker,
748
+ token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
749
+ isEncodingAsBson: false
750
+ });
751
+ const iter = stream[Symbol.asyncIterator]();
752
+ context.onTestFinished(() => {
753
+ iter.return?.();
754
+ });
755
+ // Initial empty checkpoint
756
+ const checkpoint1 = await getCheckpointLines(iter);
757
+ expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([]);
758
+ expect(checkpoint1).toMatchSnapshot();
759
+ // Add user
760
+ await writer.save({
761
+ sourceTable: usersTable,
762
+ tag: storage.SaveOperationTag.INSERT,
763
+ after: {
764
+ id: 'user1',
765
+ name: 'User 1'
766
+ },
767
+ afterReplicaId: 'user1'
768
+ });
769
+ await writer.commit('0/1');
770
+ const checkpoint2 = await getCheckpointLines(iter);
771
+ const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
772
+ expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
773
+ expect(checkpoint2).toMatchSnapshot();
880
774
  });
881
775
  test('sync updates to data query only', async (context) => {
882
- const env_11 = { stack: [], error: void 0, hasError: false };
883
- try {
884
- const f = __addDisposableResource(env_11, await factory(), true);
885
- const syncRules = await updateSyncRules(f, {
886
- content: `bucket_definitions:
776
+ await using f = await factory();
777
+ const syncRules = await updateSyncRules(f, {
778
+ content: `bucket_definitions:
887
779
  by_user:
888
780
  parameters: select users.id as user_id from users where users.id = request.user_id()
889
781
  data:
890
782
  - select * from lists where user_id = bucket.user_id
891
783
  `
892
- });
893
- const bucketStorage = await f.getInstance(syncRules);
894
- const writer = __addDisposableResource(env_11, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
895
- const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
896
- const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
897
- await writer.markAllSnapshotDone('0/1');
898
- await writer.save({
899
- sourceTable: usersTable,
900
- tag: storage.SaveOperationTag.INSERT,
901
- after: {
902
- id: 'user1',
903
- name: 'User 1'
904
- },
905
- afterReplicaId: 'user1'
906
- });
907
- await writer.commit('0/1');
908
- const stream = sync.streamResponse({
909
- syncContext,
910
- bucketStorage,
911
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
912
- params: {
913
- buckets: [],
914
- include_checksum: true,
915
- raw_data: true
916
- },
917
- tracker,
918
- token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
919
- isEncodingAsBson: false
920
- });
921
- const iter = stream[Symbol.asyncIterator]();
922
- context.onTestFinished(() => {
923
- iter.return?.();
924
- });
925
- const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
926
- const checkpoint1 = await getCheckpointLines(iter);
927
- expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
928
- expect(checkpoint1).toMatchSnapshot();
929
- await writer.save({
930
- sourceTable: listsTable,
931
- tag: storage.SaveOperationTag.INSERT,
932
- after: {
933
- id: 'list1',
934
- user_id: 'user1',
935
- name: 'User 1'
936
- },
937
- afterReplicaId: 'list1'
938
- });
939
- await writer.commit('0/1');
940
- const checkpoint2 = await getCheckpointLines(iter);
941
- expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
942
- expect(checkpoint2).toMatchSnapshot();
943
- }
944
- catch (e_11) {
945
- env_11.error = e_11;
946
- env_11.hasError = true;
947
- }
948
- finally {
949
- const result_11 = __disposeResources(env_11);
950
- if (result_11)
951
- await result_11;
952
- }
784
+ });
785
+ const bucketStorage = await f.getInstance(syncRules);
786
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
787
+ const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
788
+ const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
789
+ await writer.markAllSnapshotDone('0/1');
790
+ await writer.save({
791
+ sourceTable: usersTable,
792
+ tag: storage.SaveOperationTag.INSERT,
793
+ after: {
794
+ id: 'user1',
795
+ name: 'User 1'
796
+ },
797
+ afterReplicaId: 'user1'
798
+ });
799
+ await writer.commit('0/1');
800
+ const stream = sync.streamResponse({
801
+ syncContext,
802
+ bucketStorage,
803
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
804
+ params: {
805
+ buckets: [],
806
+ include_checksum: true,
807
+ raw_data: true
808
+ },
809
+ tracker,
810
+ token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
811
+ isEncodingAsBson: false
812
+ });
813
+ const iter = stream[Symbol.asyncIterator]();
814
+ context.onTestFinished(() => {
815
+ iter.return?.();
816
+ });
817
+ const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
818
+ const checkpoint1 = await getCheckpointLines(iter);
819
+ expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
820
+ expect(checkpoint1).toMatchSnapshot();
821
+ await writer.save({
822
+ sourceTable: listsTable,
823
+ tag: storage.SaveOperationTag.INSERT,
824
+ after: {
825
+ id: 'list1',
826
+ user_id: 'user1',
827
+ name: 'User 1'
828
+ },
829
+ afterReplicaId: 'list1'
830
+ });
831
+ await writer.commit('0/1');
832
+ const checkpoint2 = await getCheckpointLines(iter);
833
+ expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
834
+ expect(checkpoint2).toMatchSnapshot();
953
835
  });
954
836
  test('sync updates to parameter query + data', async (context) => {
955
- const env_12 = { stack: [], error: void 0, hasError: false };
956
- try {
957
- const f = __addDisposableResource(env_12, await factory(), true);
958
- const syncRules = await updateSyncRules(f, {
959
- content: `bucket_definitions:
837
+ await using f = await factory();
838
+ const syncRules = await updateSyncRules(f, {
839
+ content: `bucket_definitions:
960
840
  by_user:
961
841
  parameters: select users.id as user_id from users where users.id = request.user_id()
962
842
  data:
963
843
  - select * from lists where user_id = bucket.user_id
964
844
  `
965
- });
966
- const bucketStorage = await f.getInstance(syncRules);
967
- const writer = __addDisposableResource(env_12, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
968
- const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
969
- const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
970
- // Activate
971
- await writer.markAllSnapshotDone('0/0');
972
- await writer.keepalive('0/0');
973
- const stream = sync.streamResponse({
974
- syncContext,
975
- bucketStorage,
976
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
977
- params: {
978
- buckets: [],
979
- include_checksum: true,
980
- raw_data: true
981
- },
982
- tracker,
983
- token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
984
- isEncodingAsBson: false
985
- });
986
- const iter = stream[Symbol.asyncIterator]();
987
- context.onTestFinished(() => {
988
- iter.return?.();
989
- });
990
- // Initial empty checkpoint
991
- expect(await getCheckpointLines(iter)).toMatchSnapshot();
992
- await writer.markAllSnapshotDone('0/1');
993
- await writer.save({
994
- sourceTable: listsTable,
995
- tag: storage.SaveOperationTag.INSERT,
996
- after: {
997
- id: 'list1',
998
- user_id: 'user1',
999
- name: 'User 1'
1000
- },
1001
- afterReplicaId: 'list1'
1002
- });
1003
- await writer.save({
1004
- sourceTable: usersTable,
1005
- tag: storage.SaveOperationTag.INSERT,
1006
- after: {
1007
- id: 'user1',
1008
- name: 'User 1'
1009
- },
1010
- afterReplicaId: 'user1'
1011
- });
1012
- await writer.commit('0/1');
1013
- const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
1014
- const checkpoint2 = await getCheckpointLines(iter);
1015
- expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
1016
- expect(checkpoint2).toMatchSnapshot();
1017
- }
1018
- catch (e_12) {
1019
- env_12.error = e_12;
1020
- env_12.hasError = true;
1021
- }
1022
- finally {
1023
- const result_12 = __disposeResources(env_12);
1024
- if (result_12)
1025
- await result_12;
1026
- }
845
+ });
846
+ const bucketStorage = await f.getInstance(syncRules);
847
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
848
+ const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
849
+ const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
850
+ // Activate
851
+ await writer.markAllSnapshotDone('0/0');
852
+ await writer.keepalive('0/0');
853
+ const stream = sync.streamResponse({
854
+ syncContext,
855
+ bucketStorage,
856
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
857
+ params: {
858
+ buckets: [],
859
+ include_checksum: true,
860
+ raw_data: true
861
+ },
862
+ tracker,
863
+ token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
864
+ isEncodingAsBson: false
865
+ });
866
+ const iter = stream[Symbol.asyncIterator]();
867
+ context.onTestFinished(() => {
868
+ iter.return?.();
869
+ });
870
+ // Initial empty checkpoint
871
+ expect(await getCheckpointLines(iter)).toMatchSnapshot();
872
+ await writer.markAllSnapshotDone('0/1');
873
+ await writer.save({
874
+ sourceTable: listsTable,
875
+ tag: storage.SaveOperationTag.INSERT,
876
+ after: {
877
+ id: 'list1',
878
+ user_id: 'user1',
879
+ name: 'User 1'
880
+ },
881
+ afterReplicaId: 'list1'
882
+ });
883
+ await writer.save({
884
+ sourceTable: usersTable,
885
+ tag: storage.SaveOperationTag.INSERT,
886
+ after: {
887
+ id: 'user1',
888
+ name: 'User 1'
889
+ },
890
+ afterReplicaId: 'user1'
891
+ });
892
+ await writer.commit('0/1');
893
+ const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
894
+ const checkpoint2 = await getCheckpointLines(iter);
895
+ expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
896
+ expect(checkpoint2).toMatchSnapshot();
1027
897
  });
1028
898
  test('expiring token', async (context) => {
1029
- const env_13 = { stack: [], error: void 0, hasError: false };
1030
- try {
1031
- const f = __addDisposableResource(env_13, await factory(), true);
1032
- const syncRules = await updateSyncRules(f, {
1033
- content: BASIC_SYNC_RULES
1034
- });
1035
- const bucketStorage = await f.getInstance(syncRules);
1036
- const writer = __addDisposableResource(env_13, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1037
- // Activate
1038
- await writer.markAllSnapshotDone('0/0');
1039
- await writer.keepalive('0/0');
1040
- const exp = Date.now() / 1000 + 0.1;
1041
- const stream = sync.streamResponse({
1042
- syncContext,
1043
- bucketStorage,
1044
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1045
- params: {
1046
- buckets: [],
1047
- include_checksum: true,
1048
- raw_data: true
1049
- },
1050
- tracker,
1051
- token: new JwtPayload({ sub: '', exp: exp }),
1052
- isEncodingAsBson: false
1053
- });
1054
- const iter = stream[Symbol.asyncIterator]();
1055
- context.onTestFinished(() => {
1056
- iter.return?.();
1057
- });
1058
- const checkpoint = await getCheckpointLines(iter);
1059
- expect(checkpoint).toMatchSnapshot();
1060
- const expLines = await getCheckpointLines(iter);
1061
- expect(expLines).toMatchSnapshot();
1062
- }
1063
- catch (e_13) {
1064
- env_13.error = e_13;
1065
- env_13.hasError = true;
1066
- }
1067
- finally {
1068
- const result_13 = __disposeResources(env_13);
1069
- if (result_13)
1070
- await result_13;
1071
- }
899
+ await using f = await factory();
900
+ const syncRules = await updateSyncRules(f, {
901
+ content: BASIC_SYNC_RULES
902
+ });
903
+ const bucketStorage = await f.getInstance(syncRules);
904
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
905
+ // Activate
906
+ await writer.markAllSnapshotDone('0/0');
907
+ await writer.keepalive('0/0');
908
+ const exp = Date.now() / 1000 + 0.1;
909
+ const stream = sync.streamResponse({
910
+ syncContext,
911
+ bucketStorage,
912
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
913
+ params: {
914
+ buckets: [],
915
+ include_checksum: true,
916
+ raw_data: true
917
+ },
918
+ tracker,
919
+ token: new JwtPayload({ sub: '', exp: exp }),
920
+ isEncodingAsBson: false
921
+ });
922
+ const iter = stream[Symbol.asyncIterator]();
923
+ context.onTestFinished(() => {
924
+ iter.return?.();
925
+ });
926
+ const checkpoint = await getCheckpointLines(iter);
927
+ expect(checkpoint).toMatchSnapshot();
928
+ const expLines = await getCheckpointLines(iter);
929
+ expect(expLines).toMatchSnapshot();
1072
930
  });
1073
931
  test('compacting data - invalidate checkpoint', async (context) => {
1074
- const env_14 = { stack: [], error: void 0, hasError: false };
1075
- try {
1076
- // This tests a case of a compact operation invalidating a checkpoint in the
1077
- // middle of syncing data.
1078
- // This is expected to be rare in practice, but it is important to handle
1079
- // this case correctly to maintain consistency on the client.
1080
- const f = __addDisposableResource(env_14, await factory(), true);
932
+ // This tests a case of a compact operation invalidating a checkpoint in the
933
+ // middle of syncing data.
934
+ // This is expected to be rare in practice, but it is important to handle
935
+ // this case correctly to maintain consistency on the client.
936
+ await using f = await factory();
937
+ const syncRules = await updateSyncRules(f, {
938
+ content: BASIC_SYNC_RULES
939
+ });
940
+ const bucketStorage = await f.getInstance(syncRules);
941
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
942
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
943
+ await writer.markAllSnapshotDone('0/1');
944
+ await writer.save({
945
+ sourceTable: testTable,
946
+ tag: storage.SaveOperationTag.INSERT,
947
+ after: {
948
+ id: 't1',
949
+ description: 'Test 1'
950
+ },
951
+ afterReplicaId: 't1'
952
+ });
953
+ await writer.save({
954
+ sourceTable: testTable,
955
+ tag: storage.SaveOperationTag.INSERT,
956
+ after: {
957
+ id: 't2',
958
+ description: 'Test 2'
959
+ },
960
+ afterReplicaId: 't2'
961
+ });
962
+ await writer.commit('0/1');
963
+ const stream = sync.streamResponse({
964
+ syncContext,
965
+ bucketStorage,
966
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
967
+ params: {
968
+ buckets: [],
969
+ include_checksum: true,
970
+ raw_data: true
971
+ },
972
+ tracker,
973
+ token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
974
+ isEncodingAsBson: false
975
+ });
976
+ const iter = stream[Symbol.asyncIterator]();
977
+ context.onTestFinished(() => {
978
+ iter.return?.();
979
+ });
980
+ // Only consume the first "checkpoint" message, and pause before receiving data.
981
+ const lines = await consumeIterator(iter, { consume: false, isDone: (line) => line?.checkpoint != null });
982
+ expect(lines).toMatchSnapshot();
983
+ expect(lines[0]).toEqual({
984
+ checkpoint: expect.objectContaining({
985
+ last_op_id: '2'
986
+ })
987
+ });
988
+ // Now we save additional data AND compact before continuing.
989
+ // This invalidates the checkpoint we've received above.
990
+ await writer.markAllSnapshotDone('0/1');
991
+ await writer.save({
992
+ sourceTable: testTable,
993
+ tag: storage.SaveOperationTag.UPDATE,
994
+ after: {
995
+ id: 't1',
996
+ description: 'Test 1b'
997
+ },
998
+ afterReplicaId: 't1'
999
+ });
1000
+ await writer.save({
1001
+ sourceTable: testTable,
1002
+ tag: storage.SaveOperationTag.UPDATE,
1003
+ after: {
1004
+ id: 't2',
1005
+ description: 'Test 2b'
1006
+ },
1007
+ afterReplicaId: 't2'
1008
+ });
1009
+ await writer.commit('0/2');
1010
+ await bucketStorage.compact({
1011
+ minBucketChanges: 1,
1012
+ minChangeRatio: 0
1013
+ });
1014
+ const lines2 = await getCheckpointLines(iter, { consume: true });
1015
+ // Snapshot test checks for changes in general.
1016
+ // The tests after that documents the specific things we're looking for
1017
+ // in this test.
1018
+ expect(lines2).toMatchSnapshot();
1019
+ expect(lines2[0]).toEqual({
1020
+ data: expect.objectContaining({
1021
+ has_more: false,
1022
+ data: [
1023
+ // The first two ops have been replaced by a single CLEAR op
1024
+ expect.objectContaining({
1025
+ op: 'CLEAR'
1026
+ })
1027
+ ]
1028
+ })
1029
+ });
1030
+ // Note: No checkpoint_complete here, since the checkpoint has been
1031
+ // invalidated by the CLEAR op.
1032
+ expect(lines2[1]).toEqual({
1033
+ checkpoint_diff: expect.objectContaining({
1034
+ last_op_id: '4'
1035
+ })
1036
+ });
1037
+ expect(lines2[2]).toEqual({
1038
+ data: expect.objectContaining({
1039
+ has_more: false,
1040
+ data: [
1041
+ expect.objectContaining({
1042
+ op: 'PUT'
1043
+ }),
1044
+ expect.objectContaining({
1045
+ op: 'PUT'
1046
+ })
1047
+ ]
1048
+ })
1049
+ });
1050
+ // Now we get a checkpoint_complete
1051
+ expect(lines2[3]).toEqual({
1052
+ checkpoint_complete: expect.objectContaining({
1053
+ last_op_id: '4'
1054
+ })
1055
+ });
1056
+ });
1057
+ test('write checkpoint', async () => {
1058
+ await using f = await factory();
1059
+ const syncRules = await updateSyncRules(f, {
1060
+ content: BASIC_SYNC_RULES
1061
+ });
1062
+ const bucketStorage = f.getInstance(syncRules);
1063
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1064
+ await writer.markAllSnapshotDone('0/1');
1065
+ // <= the managed write checkpoint LSN below
1066
+ await writer.commit('0/1');
1067
+ const checkpoint = await bucketStorage.createManagedWriteCheckpoint({
1068
+ user_id: 'test',
1069
+ heads: { '1': '1/0' }
1070
+ });
1071
+ const params = {
1072
+ syncContext,
1073
+ bucketStorage,
1074
+ syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1075
+ params: {
1076
+ buckets: [],
1077
+ include_checksum: true,
1078
+ raw_data: true
1079
+ },
1080
+ tracker,
1081
+ token: new JwtPayload({ sub: 'test', exp: Date.now() / 1000 + 10 }),
1082
+ isEncodingAsBson: false
1083
+ };
1084
+ const stream1 = sync.streamResponse(params);
1085
+ const lines1 = await consumeCheckpointLines(stream1);
1086
+ // If write checkpoints are not correctly filtered, this may already
1087
+ // contain the write checkpoint.
1088
+ expect(lines1[0]).toMatchObject({
1089
+ checkpoint: expect.objectContaining({
1090
+ last_op_id: '0',
1091
+ write_checkpoint: undefined
1092
+ })
1093
+ });
1094
+ await writer.markAllSnapshotDone('0/1');
1095
+ // must be >= the managed write checkpoint LSN
1096
+ await writer.commit('1/0');
1097
+ // At this point the LSN has advanced, so the write checkpoint should be
1098
+ // included in the next checkpoint message.
1099
+ const stream2 = sync.streamResponse(params);
1100
+ const lines2 = await consumeCheckpointLines(stream2);
1101
+ expect(lines2[0]).toMatchObject({
1102
+ checkpoint: expect.objectContaining({
1103
+ last_op_id: '0',
1104
+ write_checkpoint: `${checkpoint}`
1105
+ })
1106
+ });
1107
+ });
1108
+ test('encodes sync rules id in buckets for streams', async () => {
1109
+ await using f = await factory();
1110
+ // This test relies making an actual update to sync rules to test the different bucket names.
1111
+ // The actual naming scheme may change, as long as the two buckets have different names.
1112
+ const rules = [
1113
+ `
1114
+ streams:
1115
+ test:
1116
+ auto_subscribe: true
1117
+ query: SELECT * FROM test;
1118
+
1119
+ config:
1120
+ edition: 2
1121
+ `,
1122
+ `
1123
+ streams:
1124
+ test2:
1125
+ auto_subscribe: true
1126
+ query: SELECT * FROM test WHERE 1;
1127
+
1128
+ config:
1129
+ edition: 2
1130
+ `
1131
+ ];
1132
+ for (let i = 0; i < 2; i++) {
1081
1133
  const syncRules = await updateSyncRules(f, {
1082
- content: BASIC_SYNC_RULES
1134
+ content: rules[i]
1083
1135
  });
1084
- const bucketStorage = await f.getInstance(syncRules);
1085
- const writer = __addDisposableResource(env_14, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1086
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
1136
+ const bucketStorage = f.getInstance(syncRules);
1137
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1138
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1);
1087
1139
  await writer.markAllSnapshotDone('0/1');
1088
1140
  await writer.save({
1089
1141
  sourceTable: testTable,
@@ -1094,19 +1146,10 @@ bucket_definitions:
1094
1146
  },
1095
1147
  afterReplicaId: 't1'
1096
1148
  });
1097
- await writer.save({
1098
- sourceTable: testTable,
1099
- tag: storage.SaveOperationTag.INSERT,
1100
- after: {
1101
- id: 't2',
1102
- description: 'Test 2'
1103
- },
1104
- afterReplicaId: 't2'
1105
- });
1106
1149
  await writer.commit('0/1');
1107
1150
  const stream = sync.streamResponse({
1108
1151
  syncContext,
1109
- bucketStorage,
1152
+ bucketStorage: bucketStorage,
1110
1153
  syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1111
1154
  params: {
1112
1155
  buckets: [],
@@ -1117,241 +1160,8 @@ bucket_definitions:
1117
1160
  token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
1118
1161
  isEncodingAsBson: false
1119
1162
  });
1120
- const iter = stream[Symbol.asyncIterator]();
1121
- context.onTestFinished(() => {
1122
- iter.return?.();
1123
- });
1124
- // Only consume the first "checkpoint" message, and pause before receiving data.
1125
- const lines = await consumeIterator(iter, { consume: false, isDone: (line) => line?.checkpoint != null });
1163
+ const lines = await consumeCheckpointLines(stream);
1126
1164
  expect(lines).toMatchSnapshot();
1127
- expect(lines[0]).toEqual({
1128
- checkpoint: expect.objectContaining({
1129
- last_op_id: '2'
1130
- })
1131
- });
1132
- // Now we save additional data AND compact before continuing.
1133
- // This invalidates the checkpoint we've received above.
1134
- await writer.markAllSnapshotDone('0/1');
1135
- await writer.save({
1136
- sourceTable: testTable,
1137
- tag: storage.SaveOperationTag.UPDATE,
1138
- after: {
1139
- id: 't1',
1140
- description: 'Test 1b'
1141
- },
1142
- afterReplicaId: 't1'
1143
- });
1144
- await writer.save({
1145
- sourceTable: testTable,
1146
- tag: storage.SaveOperationTag.UPDATE,
1147
- after: {
1148
- id: 't2',
1149
- description: 'Test 2b'
1150
- },
1151
- afterReplicaId: 't2'
1152
- });
1153
- await writer.commit('0/2');
1154
- await bucketStorage.compact({
1155
- minBucketChanges: 1,
1156
- minChangeRatio: 0
1157
- });
1158
- const lines2 = await getCheckpointLines(iter, { consume: true });
1159
- // Snapshot test checks for changes in general.
1160
- // The tests after that documents the specific things we're looking for
1161
- // in this test.
1162
- expect(lines2).toMatchSnapshot();
1163
- expect(lines2[0]).toEqual({
1164
- data: expect.objectContaining({
1165
- has_more: false,
1166
- data: [
1167
- // The first two ops have been replaced by a single CLEAR op
1168
- expect.objectContaining({
1169
- op: 'CLEAR'
1170
- })
1171
- ]
1172
- })
1173
- });
1174
- // Note: No checkpoint_complete here, since the checkpoint has been
1175
- // invalidated by the CLEAR op.
1176
- expect(lines2[1]).toEqual({
1177
- checkpoint_diff: expect.objectContaining({
1178
- last_op_id: '4'
1179
- })
1180
- });
1181
- expect(lines2[2]).toEqual({
1182
- data: expect.objectContaining({
1183
- has_more: false,
1184
- data: [
1185
- expect.objectContaining({
1186
- op: 'PUT'
1187
- }),
1188
- expect.objectContaining({
1189
- op: 'PUT'
1190
- })
1191
- ]
1192
- })
1193
- });
1194
- // Now we get a checkpoint_complete
1195
- expect(lines2[3]).toEqual({
1196
- checkpoint_complete: expect.objectContaining({
1197
- last_op_id: '4'
1198
- })
1199
- });
1200
- }
1201
- catch (e_14) {
1202
- env_14.error = e_14;
1203
- env_14.hasError = true;
1204
- }
1205
- finally {
1206
- const result_14 = __disposeResources(env_14);
1207
- if (result_14)
1208
- await result_14;
1209
- }
1210
- });
1211
- test('write checkpoint', async () => {
1212
- const env_15 = { stack: [], error: void 0, hasError: false };
1213
- try {
1214
- const f = __addDisposableResource(env_15, await factory(), true);
1215
- const syncRules = await updateSyncRules(f, {
1216
- content: BASIC_SYNC_RULES
1217
- });
1218
- const bucketStorage = f.getInstance(syncRules);
1219
- const writer = __addDisposableResource(env_15, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1220
- await writer.markAllSnapshotDone('0/1');
1221
- // <= the managed write checkpoint LSN below
1222
- await writer.commit('0/1');
1223
- const checkpoint = await bucketStorage.createManagedWriteCheckpoint({
1224
- user_id: 'test',
1225
- heads: { '1': '1/0' }
1226
- });
1227
- const params = {
1228
- syncContext,
1229
- bucketStorage,
1230
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1231
- params: {
1232
- buckets: [],
1233
- include_checksum: true,
1234
- raw_data: true
1235
- },
1236
- tracker,
1237
- token: new JwtPayload({ sub: 'test', exp: Date.now() / 1000 + 10 }),
1238
- isEncodingAsBson: false
1239
- };
1240
- const stream1 = sync.streamResponse(params);
1241
- const lines1 = await consumeCheckpointLines(stream1);
1242
- // If write checkpoints are not correctly filtered, this may already
1243
- // contain the write checkpoint.
1244
- expect(lines1[0]).toMatchObject({
1245
- checkpoint: expect.objectContaining({
1246
- last_op_id: '0',
1247
- write_checkpoint: undefined
1248
- })
1249
- });
1250
- await writer.markAllSnapshotDone('0/1');
1251
- // must be >= the managed write checkpoint LSN
1252
- await writer.commit('1/0');
1253
- // At this point the LSN has advanced, so the write checkpoint should be
1254
- // included in the next checkpoint message.
1255
- const stream2 = sync.streamResponse(params);
1256
- const lines2 = await consumeCheckpointLines(stream2);
1257
- expect(lines2[0]).toMatchObject({
1258
- checkpoint: expect.objectContaining({
1259
- last_op_id: '0',
1260
- write_checkpoint: `${checkpoint}`
1261
- })
1262
- });
1263
- }
1264
- catch (e_15) {
1265
- env_15.error = e_15;
1266
- env_15.hasError = true;
1267
- }
1268
- finally {
1269
- const result_15 = __disposeResources(env_15);
1270
- if (result_15)
1271
- await result_15;
1272
- }
1273
- });
1274
- test('encodes sync rules id in buckets for streams', async () => {
1275
- const env_16 = { stack: [], error: void 0, hasError: false };
1276
- try {
1277
- const f = __addDisposableResource(env_16, await factory(), true);
1278
- // This test relies making an actual update to sync rules to test the different bucket names.
1279
- // The actual naming scheme may change, as long as the two buckets have different names.
1280
- const rules = [
1281
- `
1282
- streams:
1283
- test:
1284
- auto_subscribe: true
1285
- query: SELECT * FROM test;
1286
-
1287
- config:
1288
- edition: 2
1289
- `,
1290
- `
1291
- streams:
1292
- test2:
1293
- auto_subscribe: true
1294
- query: SELECT * FROM test WHERE 1;
1295
-
1296
- config:
1297
- edition: 2
1298
- `
1299
- ];
1300
- for (let i = 0; i < 2; i++) {
1301
- const env_17 = { stack: [], error: void 0, hasError: false };
1302
- try {
1303
- const syncRules = await updateSyncRules(f, {
1304
- content: rules[i]
1305
- });
1306
- const bucketStorage = f.getInstance(syncRules);
1307
- const writer = __addDisposableResource(env_17, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
1308
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1);
1309
- await writer.markAllSnapshotDone('0/1');
1310
- await writer.save({
1311
- sourceTable: testTable,
1312
- tag: storage.SaveOperationTag.INSERT,
1313
- after: {
1314
- id: 't1',
1315
- description: 'Test 1'
1316
- },
1317
- afterReplicaId: 't1'
1318
- });
1319
- await writer.commit('0/1');
1320
- const stream = sync.streamResponse({
1321
- syncContext,
1322
- bucketStorage: bucketStorage,
1323
- syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
1324
- params: {
1325
- buckets: [],
1326
- include_checksum: true,
1327
- raw_data: true
1328
- },
1329
- tracker,
1330
- token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
1331
- isEncodingAsBson: false
1332
- });
1333
- const lines = await consumeCheckpointLines(stream);
1334
- expect(lines).toMatchSnapshot();
1335
- }
1336
- catch (e_16) {
1337
- env_17.error = e_16;
1338
- env_17.hasError = true;
1339
- }
1340
- finally {
1341
- const result_16 = __disposeResources(env_17);
1342
- if (result_16)
1343
- await result_16;
1344
- }
1345
- }
1346
- }
1347
- catch (e_17) {
1348
- env_16.error = e_17;
1349
- env_16.hasError = true;
1350
- }
1351
- finally {
1352
- const result_17 = __disposeResources(env_16);
1353
- if (result_17)
1354
- await result_17;
1355
1165
  }
1356
1166
  });
1357
1167
  }