@powersync/service-core-tests 0.15.2 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/test-utils/MetricsHelper.js +1 -1
- package/dist/test-utils/MetricsHelper.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +491 -627
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +192 -304
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +1125 -1446
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +452 -636
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-migration-tests.js +63 -139
- package/dist/tests/register-migration-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +94 -170
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +965 -1155
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/dist/tests/tests-index.d.ts +4 -4
- package/dist/tests/tests-index.js +4 -4
- package/dist/tests/tests-index.js.map +1 -1
- package/package.json +5 -5
- package/src/test-utils/MetricsHelper.ts +1 -1
- package/src/tests/register-data-storage-data-tests.ts +22 -0
- package/src/tests/register-parameter-compacting-tests.ts +0 -1
- package/src/tests/register-sync-tests.ts +74 -0
- package/src/tests/tests-index.ts +4 -4
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,55 +1,3 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
1
|
import { createCoreAPIMetrics, JwtPayload, storage, sync, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
54
2
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
55
3
|
import path from 'path';
|
|
@@ -94,67 +42,53 @@ export function registerSyncTests(configOrFactory, options = {}) {
|
|
|
94
42
|
}));
|
|
95
43
|
};
|
|
96
44
|
test('sync global data', async () => {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
const lines = await consumeCheckpointLines(stream);
|
|
140
|
-
expect(lines).toMatchSnapshot();
|
|
141
|
-
}
|
|
142
|
-
catch (e_1) {
|
|
143
|
-
env_1.error = e_1;
|
|
144
|
-
env_1.hasError = true;
|
|
145
|
-
}
|
|
146
|
-
finally {
|
|
147
|
-
const result_1 = __disposeResources(env_1);
|
|
148
|
-
if (result_1)
|
|
149
|
-
await result_1;
|
|
150
|
-
}
|
|
45
|
+
await using f = await factory();
|
|
46
|
+
const syncRules = await updateSyncRules(f, {
|
|
47
|
+
content: BASIC_SYNC_RULES
|
|
48
|
+
});
|
|
49
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
50
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
51
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
52
|
+
await writer.markAllSnapshotDone('0/1');
|
|
53
|
+
await writer.save({
|
|
54
|
+
sourceTable,
|
|
55
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
56
|
+
after: {
|
|
57
|
+
id: 't1',
|
|
58
|
+
description: 'Test 1'
|
|
59
|
+
},
|
|
60
|
+
afterReplicaId: 't1'
|
|
61
|
+
});
|
|
62
|
+
await writer.save({
|
|
63
|
+
sourceTable,
|
|
64
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
65
|
+
after: {
|
|
66
|
+
id: 't2',
|
|
67
|
+
description: 'Test 2'
|
|
68
|
+
},
|
|
69
|
+
afterReplicaId: 't2'
|
|
70
|
+
});
|
|
71
|
+
await writer.commit('0/1');
|
|
72
|
+
const stream = sync.streamResponse({
|
|
73
|
+
syncContext,
|
|
74
|
+
bucketStorage: bucketStorage,
|
|
75
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
76
|
+
params: {
|
|
77
|
+
buckets: [],
|
|
78
|
+
include_checksum: true,
|
|
79
|
+
raw_data: true
|
|
80
|
+
},
|
|
81
|
+
tracker,
|
|
82
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
83
|
+
isEncodingAsBson: false
|
|
84
|
+
});
|
|
85
|
+
const lines = await consumeCheckpointLines(stream);
|
|
86
|
+
expect(lines).toMatchSnapshot();
|
|
151
87
|
});
|
|
152
88
|
test('sync buckets in order', async () => {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
const syncRules = await updateSyncRules(f, {
|
|
157
|
-
content: `
|
|
89
|
+
await using f = await factory();
|
|
90
|
+
const syncRules = await updateSyncRules(f, {
|
|
91
|
+
content: `
|
|
158
92
|
bucket_definitions:
|
|
159
93
|
b0:
|
|
160
94
|
priority: 2
|
|
@@ -165,62 +99,116 @@ bucket_definitions:
|
|
|
165
99
|
data:
|
|
166
100
|
- SELECT * FROM test WHERE LENGTH(id) > 2;
|
|
167
101
|
`
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
102
|
+
});
|
|
103
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
104
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
105
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
106
|
+
await writer.markAllSnapshotDone('0/1');
|
|
107
|
+
await writer.save({
|
|
108
|
+
sourceTable: testTable,
|
|
109
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
110
|
+
after: {
|
|
111
|
+
id: 't1',
|
|
112
|
+
description: 'Test 1'
|
|
113
|
+
},
|
|
114
|
+
afterReplicaId: 't1'
|
|
115
|
+
});
|
|
116
|
+
await writer.save({
|
|
117
|
+
sourceTable: testTable,
|
|
118
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
119
|
+
after: {
|
|
120
|
+
id: 'earlier',
|
|
121
|
+
description: 'Test 2'
|
|
122
|
+
},
|
|
123
|
+
afterReplicaId: 'earlier'
|
|
124
|
+
});
|
|
125
|
+
await writer.commit('0/1');
|
|
126
|
+
const stream = sync.streamResponse({
|
|
127
|
+
syncContext,
|
|
128
|
+
bucketStorage,
|
|
129
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
130
|
+
params: {
|
|
131
|
+
buckets: [],
|
|
132
|
+
include_checksum: true,
|
|
133
|
+
raw_data: true
|
|
134
|
+
},
|
|
135
|
+
tracker,
|
|
136
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
137
|
+
isEncodingAsBson: false
|
|
138
|
+
});
|
|
139
|
+
const lines = await consumeCheckpointLines(stream);
|
|
140
|
+
expect(lines).toMatchSnapshot();
|
|
141
|
+
});
|
|
142
|
+
test('can override priority when subscribing to stream', async () => {
|
|
143
|
+
await using f = await factory();
|
|
144
|
+
const syncRules = await updateSyncRules(f, {
|
|
145
|
+
content: `
|
|
146
|
+
config:
|
|
147
|
+
edition: 3
|
|
148
|
+
|
|
149
|
+
streams:
|
|
150
|
+
todos:
|
|
151
|
+
query: SELECT * FROM test WHERE id IN subscription.parameter('test_ids')
|
|
152
|
+
`
|
|
153
|
+
});
|
|
154
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
155
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
156
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
157
|
+
await writer.markAllSnapshotDone('0/1');
|
|
158
|
+
await writer.save({
|
|
159
|
+
sourceTable: testTable,
|
|
160
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
161
|
+
after: {
|
|
162
|
+
id: 'a',
|
|
163
|
+
description: 'Test 1'
|
|
164
|
+
},
|
|
165
|
+
afterReplicaId: 't1'
|
|
166
|
+
});
|
|
167
|
+
await writer.save({
|
|
168
|
+
sourceTable: testTable,
|
|
169
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
170
|
+
after: {
|
|
171
|
+
id: 'b',
|
|
172
|
+
description: 'Test 2'
|
|
173
|
+
},
|
|
174
|
+
afterReplicaId: 'earlier'
|
|
175
|
+
});
|
|
176
|
+
await writer.commit('0/1');
|
|
177
|
+
const stream = sync.streamResponse({
|
|
178
|
+
syncContext,
|
|
179
|
+
bucketStorage,
|
|
180
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
181
|
+
params: {
|
|
182
|
+
buckets: [],
|
|
183
|
+
include_checksum: true,
|
|
184
|
+
raw_data: true,
|
|
185
|
+
streams: {
|
|
186
|
+
include_defaults: true,
|
|
187
|
+
subscriptions: [
|
|
188
|
+
{
|
|
189
|
+
stream: 'todos',
|
|
190
|
+
parameters: { test_ids: ['a'] },
|
|
191
|
+
override_priority: 0
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
stream: 'todos',
|
|
195
|
+
parameters: { test_ids: ['a', 'b'] },
|
|
196
|
+
override_priority: null
|
|
197
|
+
}
|
|
198
|
+
]
|
|
199
|
+
}
|
|
200
|
+
},
|
|
201
|
+
tracker,
|
|
202
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
203
|
+
isEncodingAsBson: false
|
|
204
|
+
});
|
|
205
|
+
const lines = await consumeCheckpointLines(stream);
|
|
206
|
+
expect(lines).toMatchSnapshot();
|
|
217
207
|
});
|
|
218
208
|
test('sync interrupts low-priority buckets on new checkpoints', async () => {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
const syncRules = await updateSyncRules(f, {
|
|
223
|
-
content: `
|
|
209
|
+
await using f = await factory();
|
|
210
|
+
const syncRules = await updateSyncRules(f, {
|
|
211
|
+
content: `
|
|
224
212
|
bucket_definitions:
|
|
225
213
|
b0:
|
|
226
214
|
priority: 2
|
|
@@ -231,105 +219,93 @@ bucket_definitions:
|
|
|
231
219
|
data:
|
|
232
220
|
- SELECT * FROM test WHERE LENGTH(id) > 5;
|
|
233
221
|
`
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
222
|
+
});
|
|
223
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
224
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
225
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
226
|
+
await writer.markAllSnapshotDone('0/1');
|
|
227
|
+
// Initial data: Add one priority row and 10k low-priority rows.
|
|
228
|
+
await writer.save({
|
|
229
|
+
sourceTable: testTable,
|
|
230
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
231
|
+
after: {
|
|
232
|
+
id: 'highprio',
|
|
233
|
+
description: 'High priority row'
|
|
234
|
+
},
|
|
235
|
+
afterReplicaId: 'highprio'
|
|
236
|
+
});
|
|
237
|
+
for (let i = 0; i < 10_000; i++) {
|
|
240
238
|
await writer.save({
|
|
241
239
|
sourceTable: testTable,
|
|
242
240
|
tag: storage.SaveOperationTag.INSERT,
|
|
243
241
|
after: {
|
|
244
|
-
id:
|
|
245
|
-
description: '
|
|
242
|
+
id: `${i}`,
|
|
243
|
+
description: 'low prio'
|
|
246
244
|
},
|
|
247
|
-
afterReplicaId:
|
|
245
|
+
afterReplicaId: `${i}`
|
|
248
246
|
});
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
247
|
+
}
|
|
248
|
+
await writer.commit('0/1');
|
|
249
|
+
const stream = sync.streamResponse({
|
|
250
|
+
syncContext,
|
|
251
|
+
bucketStorage,
|
|
252
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
253
|
+
params: {
|
|
254
|
+
buckets: [],
|
|
255
|
+
include_checksum: true,
|
|
256
|
+
raw_data: true
|
|
257
|
+
},
|
|
258
|
+
tracker,
|
|
259
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
260
|
+
isEncodingAsBson: false
|
|
261
|
+
});
|
|
262
|
+
let sentCheckpoints = 0;
|
|
263
|
+
let sentRows = 0;
|
|
264
|
+
for await (let next of stream) {
|
|
265
|
+
if (typeof next == 'string') {
|
|
266
|
+
next = JSON.parse(next);
|
|
259
267
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
let sentRows = 0;
|
|
276
|
-
for await (let next of stream) {
|
|
277
|
-
if (typeof next == 'string') {
|
|
278
|
-
next = JSON.parse(next);
|
|
279
|
-
}
|
|
280
|
-
if (typeof next === 'object' && next !== null) {
|
|
281
|
-
if ('partial_checkpoint_complete' in next) {
|
|
282
|
-
if (sentCheckpoints == 1) {
|
|
283
|
-
// Save new data to interrupt the low-priority sync.
|
|
284
|
-
// Add another high-priority row. This should interrupt the long-running low-priority sync.
|
|
285
|
-
await writer.save({
|
|
286
|
-
sourceTable: testTable,
|
|
287
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
288
|
-
after: {
|
|
289
|
-
id: 'highprio2',
|
|
290
|
-
description: 'Another high-priority row'
|
|
291
|
-
},
|
|
292
|
-
afterReplicaId: 'highprio2'
|
|
293
|
-
});
|
|
294
|
-
await writer.commit('0/2');
|
|
295
|
-
}
|
|
296
|
-
else {
|
|
297
|
-
// Low-priority sync from the first checkpoint was interrupted. This should not happen before
|
|
298
|
-
// 1000 low-priority items were synchronized.
|
|
299
|
-
expect(sentCheckpoints).toBe(2);
|
|
300
|
-
expect(sentRows).toBeGreaterThan(1000);
|
|
301
|
-
}
|
|
268
|
+
if (typeof next === 'object' && next !== null) {
|
|
269
|
+
if ('partial_checkpoint_complete' in next) {
|
|
270
|
+
if (sentCheckpoints == 1) {
|
|
271
|
+
// Save new data to interrupt the low-priority sync.
|
|
272
|
+
// Add another high-priority row. This should interrupt the long-running low-priority sync.
|
|
273
|
+
await writer.save({
|
|
274
|
+
sourceTable: testTable,
|
|
275
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
276
|
+
after: {
|
|
277
|
+
id: 'highprio2',
|
|
278
|
+
description: 'Another high-priority row'
|
|
279
|
+
},
|
|
280
|
+
afterReplicaId: 'highprio2'
|
|
281
|
+
});
|
|
282
|
+
await writer.commit('0/2');
|
|
302
283
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
sentRows
|
|
308
|
-
}
|
|
309
|
-
if ('checkpoint_complete' in next) {
|
|
310
|
-
break;
|
|
284
|
+
else {
|
|
285
|
+
// Low-priority sync from the first checkpoint was interrupted. This should not happen before
|
|
286
|
+
// 1000 low-priority items were synchronized.
|
|
287
|
+
expect(sentCheckpoints).toBe(2);
|
|
288
|
+
expect(sentRows).toBeGreaterThan(1000);
|
|
311
289
|
}
|
|
312
290
|
}
|
|
291
|
+
if ('checkpoint' in next || 'checkpoint_diff' in next) {
|
|
292
|
+
sentCheckpoints += 1;
|
|
293
|
+
}
|
|
294
|
+
if ('data' in next) {
|
|
295
|
+
sentRows += next.data.data.length;
|
|
296
|
+
}
|
|
297
|
+
if ('checkpoint_complete' in next) {
|
|
298
|
+
break;
|
|
299
|
+
}
|
|
313
300
|
}
|
|
314
|
-
expect(sentCheckpoints).toBe(2);
|
|
315
|
-
expect(sentRows).toBe(10002);
|
|
316
|
-
}
|
|
317
|
-
catch (e_3) {
|
|
318
|
-
env_3.error = e_3;
|
|
319
|
-
env_3.hasError = true;
|
|
320
|
-
}
|
|
321
|
-
finally {
|
|
322
|
-
const result_3 = __disposeResources(env_3);
|
|
323
|
-
if (result_3)
|
|
324
|
-
await result_3;
|
|
325
301
|
}
|
|
302
|
+
expect(sentCheckpoints).toBe(2);
|
|
303
|
+
expect(sentRows).toBe(10002);
|
|
326
304
|
});
|
|
327
305
|
test('sync interruptions with unrelated data', async () => {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
const syncRules = await updateSyncRules(f, {
|
|
332
|
-
content: `
|
|
306
|
+
await using f = await factory();
|
|
307
|
+
const syncRules = await updateSyncRules(f, {
|
|
308
|
+
content: `
|
|
333
309
|
bucket_definitions:
|
|
334
310
|
b0:
|
|
335
311
|
priority: 2
|
|
@@ -341,128 +317,116 @@ bucket_definitions:
|
|
|
341
317
|
data:
|
|
342
318
|
- SELECT * FROM test WHERE LENGTH(id) > 5 AND description = bucket.user_id;
|
|
343
319
|
`
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
320
|
+
});
|
|
321
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
322
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
323
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
324
|
+
await writer.markAllSnapshotDone('0/1');
|
|
325
|
+
// Initial data: Add one priority row and 10k low-priority rows.
|
|
326
|
+
await writer.save({
|
|
327
|
+
sourceTable: testTable,
|
|
328
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
329
|
+
after: {
|
|
330
|
+
id: 'highprio',
|
|
331
|
+
description: 'user_one'
|
|
332
|
+
},
|
|
333
|
+
afterReplicaId: 'highprio'
|
|
334
|
+
});
|
|
335
|
+
for (let i = 0; i < 10_000; i++) {
|
|
350
336
|
await writer.save({
|
|
351
337
|
sourceTable: testTable,
|
|
352
338
|
tag: storage.SaveOperationTag.INSERT,
|
|
353
339
|
after: {
|
|
354
|
-
id:
|
|
355
|
-
description: '
|
|
340
|
+
id: `${i}`,
|
|
341
|
+
description: 'low prio'
|
|
356
342
|
},
|
|
357
|
-
afterReplicaId:
|
|
343
|
+
afterReplicaId: `${i}`
|
|
358
344
|
});
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
345
|
+
}
|
|
346
|
+
await writer.commit('0/1');
|
|
347
|
+
const stream = sync.streamResponse({
|
|
348
|
+
syncContext,
|
|
349
|
+
bucketStorage,
|
|
350
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
351
|
+
params: {
|
|
352
|
+
buckets: [],
|
|
353
|
+
include_checksum: true,
|
|
354
|
+
raw_data: true
|
|
355
|
+
},
|
|
356
|
+
tracker,
|
|
357
|
+
token: new JwtPayload({ sub: 'user_one', exp: Date.now() / 1000 + 100000 }),
|
|
358
|
+
isEncodingAsBson: false
|
|
359
|
+
});
|
|
360
|
+
let sentCheckpoints = 0;
|
|
361
|
+
let completedCheckpoints = 0;
|
|
362
|
+
let sentRows = 0;
|
|
363
|
+
// Expected flow:
|
|
364
|
+
// 1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
|
|
365
|
+
// 2. We insert a new row that is not part of a bucket relevant to this stream.
|
|
366
|
+
// 3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
|
|
367
|
+
// 4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
|
|
368
|
+
for await (let next of stream) {
|
|
369
|
+
if (typeof next == 'string') {
|
|
370
|
+
next = JSON.parse(next);
|
|
369
371
|
}
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
let sentCheckpoints = 0;
|
|
385
|
-
let completedCheckpoints = 0;
|
|
386
|
-
let sentRows = 0;
|
|
387
|
-
// Expected flow:
|
|
388
|
-
// 1. Stream starts, we receive a checkpoint followed by the one high-prio row and a partial completion.
|
|
389
|
-
// 2. We insert a new row that is not part of a bucket relevant to this stream.
|
|
390
|
-
// 3. This means that no interruption happens and we receive all the low-priority data, followed by a checkpoint.
|
|
391
|
-
// 4. After the checkpoint, add a new row that _is_ relevant for this sync, which should trigger a new iteration.
|
|
392
|
-
for await (let next of stream) {
|
|
393
|
-
if (typeof next == 'string') {
|
|
394
|
-
next = JSON.parse(next);
|
|
395
|
-
}
|
|
396
|
-
if (typeof next === 'object' && next !== null) {
|
|
397
|
-
if ('partial_checkpoint_complete' in next) {
|
|
398
|
-
if (sentCheckpoints == 1) {
|
|
399
|
-
// Add a high-priority row that doesn't affect this sync stream.
|
|
400
|
-
await writer.save({
|
|
401
|
-
sourceTable: testTable,
|
|
402
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
403
|
-
after: {
|
|
404
|
-
id: 'highprio2',
|
|
405
|
-
description: 'user_two'
|
|
406
|
-
},
|
|
407
|
-
afterReplicaId: 'highprio2'
|
|
408
|
-
});
|
|
409
|
-
await writer.commit('0/2');
|
|
410
|
-
}
|
|
411
|
-
else {
|
|
412
|
-
expect(sentCheckpoints).toBe(2);
|
|
413
|
-
expect(sentRows).toBe(10002);
|
|
414
|
-
}
|
|
372
|
+
if (typeof next === 'object' && next !== null) {
|
|
373
|
+
if ('partial_checkpoint_complete' in next) {
|
|
374
|
+
if (sentCheckpoints == 1) {
|
|
375
|
+
// Add a high-priority row that doesn't affect this sync stream.
|
|
376
|
+
await writer.save({
|
|
377
|
+
sourceTable: testTable,
|
|
378
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
379
|
+
after: {
|
|
380
|
+
id: 'highprio2',
|
|
381
|
+
description: 'user_two'
|
|
382
|
+
},
|
|
383
|
+
afterReplicaId: 'highprio2'
|
|
384
|
+
});
|
|
385
|
+
await writer.commit('0/2');
|
|
415
386
|
}
|
|
416
|
-
|
|
417
|
-
sentCheckpoints
|
|
387
|
+
else {
|
|
388
|
+
expect(sentCheckpoints).toBe(2);
|
|
389
|
+
expect(sentRows).toBe(10002);
|
|
418
390
|
}
|
|
419
|
-
|
|
420
|
-
|
|
391
|
+
}
|
|
392
|
+
if ('checkpoint' in next || 'checkpoint_diff' in next) {
|
|
393
|
+
sentCheckpoints += 1;
|
|
394
|
+
}
|
|
395
|
+
if ('data' in next) {
|
|
396
|
+
sentRows += next.data.data.length;
|
|
397
|
+
}
|
|
398
|
+
if ('checkpoint_complete' in next) {
|
|
399
|
+
completedCheckpoints++;
|
|
400
|
+
if (completedCheckpoints == 2) {
|
|
401
|
+
break;
|
|
421
402
|
}
|
|
422
|
-
if (
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
description: 'user_one'
|
|
436
|
-
},
|
|
437
|
-
afterReplicaId: 'highprio3'
|
|
438
|
-
});
|
|
439
|
-
await writer.commit('0/3');
|
|
440
|
-
}
|
|
403
|
+
if (completedCheckpoints == 1) {
|
|
404
|
+
expect(sentRows).toBe(10001);
|
|
405
|
+
// Add a high-priority row that affects this sync stream.
|
|
406
|
+
await writer.save({
|
|
407
|
+
sourceTable: testTable,
|
|
408
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
409
|
+
after: {
|
|
410
|
+
id: 'highprio3',
|
|
411
|
+
description: 'user_one'
|
|
412
|
+
},
|
|
413
|
+
afterReplicaId: 'highprio3'
|
|
414
|
+
});
|
|
415
|
+
await writer.commit('0/3');
|
|
441
416
|
}
|
|
442
417
|
}
|
|
443
418
|
}
|
|
444
|
-
expect(sentCheckpoints).toBe(2);
|
|
445
|
-
expect(sentRows).toBe(10002);
|
|
446
|
-
}
|
|
447
|
-
catch (e_4) {
|
|
448
|
-
env_4.error = e_4;
|
|
449
|
-
env_4.hasError = true;
|
|
450
|
-
}
|
|
451
|
-
finally {
|
|
452
|
-
const result_4 = __disposeResources(env_4);
|
|
453
|
-
if (result_4)
|
|
454
|
-
await result_4;
|
|
455
419
|
}
|
|
420
|
+
expect(sentCheckpoints).toBe(2);
|
|
421
|
+
expect(sentRows).toBe(10002);
|
|
456
422
|
});
|
|
457
423
|
test('sync interrupts low-priority buckets on new checkpoints (2)', async () => {
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
const syncRules = await updateSyncRules(f, {
|
|
465
|
-
content: `
|
|
424
|
+
await using f = await factory();
|
|
425
|
+
// bucket0a -> send all data
|
|
426
|
+
// then interrupt checkpoint with new data for all buckets
|
|
427
|
+
// -> data for all buckets should be sent in the new checkpoint
|
|
428
|
+
const syncRules = await updateSyncRules(f, {
|
|
429
|
+
content: `
|
|
466
430
|
bucket_definitions:
|
|
467
431
|
b0a:
|
|
468
432
|
priority: 2
|
|
@@ -477,613 +441,701 @@ bucket_definitions:
|
|
|
477
441
|
data:
|
|
478
442
|
- SELECT * FROM test WHERE LENGTH(id) > 5;
|
|
479
443
|
`
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
444
|
+
});
|
|
445
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
446
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
447
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
448
|
+
await writer.markAllSnapshotDone('0/1');
|
|
449
|
+
// Initial data: Add one priority row and 10k low-priority rows.
|
|
450
|
+
await writer.save({
|
|
451
|
+
sourceTable: testTable,
|
|
452
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
453
|
+
after: {
|
|
454
|
+
id: 'highprio',
|
|
455
|
+
description: 'High priority row'
|
|
456
|
+
},
|
|
457
|
+
afterReplicaId: 'highprio'
|
|
458
|
+
});
|
|
459
|
+
for (let i = 0; i < 2_000; i++) {
|
|
486
460
|
await writer.save({
|
|
487
461
|
sourceTable: testTable,
|
|
488
462
|
tag: storage.SaveOperationTag.INSERT,
|
|
489
463
|
after: {
|
|
490
|
-
id:
|
|
491
|
-
description: '
|
|
464
|
+
id: `${i}`,
|
|
465
|
+
description: 'low prio'
|
|
492
466
|
},
|
|
493
|
-
afterReplicaId:
|
|
467
|
+
afterReplicaId: `${i}`
|
|
494
468
|
});
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
469
|
+
}
|
|
470
|
+
await writer.commit('0/1');
|
|
471
|
+
const stream = sync.streamResponse({
|
|
472
|
+
syncContext,
|
|
473
|
+
bucketStorage,
|
|
474
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
475
|
+
params: {
|
|
476
|
+
buckets: [],
|
|
477
|
+
include_checksum: true,
|
|
478
|
+
raw_data: true
|
|
479
|
+
},
|
|
480
|
+
tracker,
|
|
481
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
482
|
+
isEncodingAsBson: false
|
|
483
|
+
});
|
|
484
|
+
let sentRows = 0;
|
|
485
|
+
let lines = [];
|
|
486
|
+
for await (let next of stream) {
|
|
487
|
+
if (typeof next == 'string') {
|
|
488
|
+
next = JSON.parse(next);
|
|
505
489
|
}
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
bucketStorage,
|
|
510
|
-
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
511
|
-
params: {
|
|
512
|
-
buckets: [],
|
|
513
|
-
include_checksum: true,
|
|
514
|
-
raw_data: true
|
|
515
|
-
},
|
|
516
|
-
tracker,
|
|
517
|
-
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
518
|
-
isEncodingAsBson: false
|
|
519
|
-
});
|
|
520
|
-
let sentRows = 0;
|
|
521
|
-
let lines = [];
|
|
522
|
-
for await (let next of stream) {
|
|
523
|
-
if (typeof next == 'string') {
|
|
524
|
-
next = JSON.parse(next);
|
|
490
|
+
if (typeof next === 'object' && next !== null) {
|
|
491
|
+
if ('partial_checkpoint_complete' in next) {
|
|
492
|
+
lines.push(next);
|
|
525
493
|
}
|
|
526
|
-
if (
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
494
|
+
if ('checkpoint' in next || 'checkpoint_diff' in next) {
|
|
495
|
+
lines.push(next);
|
|
496
|
+
}
|
|
497
|
+
if ('data' in next) {
|
|
498
|
+
lines.push({ data: { ...next.data, data: undefined } });
|
|
499
|
+
sentRows += next.data.data.length;
|
|
500
|
+
if (sentRows == 1001) {
|
|
501
|
+
// Save new data to interrupt the low-priority sync.
|
|
502
|
+
// Add another high-priority row. This should interrupt the long-running low-priority sync.
|
|
503
|
+
await writer.save({
|
|
504
|
+
sourceTable: testTable,
|
|
505
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
506
|
+
after: {
|
|
507
|
+
id: 'highprio2',
|
|
508
|
+
description: 'Another high-priority row'
|
|
509
|
+
},
|
|
510
|
+
afterReplicaId: 'highprio2'
|
|
511
|
+
});
|
|
512
|
+
// Also add a low-priority row
|
|
513
|
+
await writer.save({
|
|
514
|
+
sourceTable: testTable,
|
|
515
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
516
|
+
after: {
|
|
517
|
+
id: '2001',
|
|
518
|
+
description: 'Another low-priority row'
|
|
519
|
+
},
|
|
520
|
+
afterReplicaId: '2001'
|
|
521
|
+
});
|
|
522
|
+
await writer.commit('0/2');
|
|
532
523
|
}
|
|
533
|
-
if (
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
// Save new data to interrupt the low-priority sync.
|
|
538
|
-
// Add another high-priority row. This should interrupt the long-running low-priority sync.
|
|
539
|
-
await writer.save({
|
|
540
|
-
sourceTable: testTable,
|
|
541
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
542
|
-
after: {
|
|
543
|
-
id: 'highprio2',
|
|
544
|
-
description: 'Another high-priority row'
|
|
545
|
-
},
|
|
546
|
-
afterReplicaId: 'highprio2'
|
|
547
|
-
});
|
|
548
|
-
// Also add a low-priority row
|
|
549
|
-
await writer.save({
|
|
550
|
-
sourceTable: testTable,
|
|
551
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
552
|
-
after: {
|
|
553
|
-
id: '2001',
|
|
554
|
-
description: 'Another low-priority row'
|
|
555
|
-
},
|
|
556
|
-
afterReplicaId: '2001'
|
|
557
|
-
});
|
|
558
|
-
await writer.commit('0/2');
|
|
559
|
-
}
|
|
560
|
-
if (sentRows >= 1000 && sentRows <= 2001) {
|
|
561
|
-
// pause for a bit to give the stream time to process interruptions.
|
|
562
|
-
// This covers the data batch above and the next one.
|
|
563
|
-
await timers.setTimeout(50);
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
if ('checkpoint_complete' in next) {
|
|
567
|
-
lines.push(next);
|
|
568
|
-
break;
|
|
524
|
+
if (sentRows >= 1000 && sentRows <= 2001) {
|
|
525
|
+
// pause for a bit to give the stream time to process interruptions.
|
|
526
|
+
// This covers the data batch above and the next one.
|
|
527
|
+
await timers.setTimeout(50);
|
|
569
528
|
}
|
|
570
529
|
}
|
|
530
|
+
if ('checkpoint_complete' in next) {
|
|
531
|
+
lines.push(next);
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
571
534
|
}
|
|
572
|
-
// Expected lines (full details in snapshot):
|
|
573
|
-
//
|
|
574
|
-
// checkpoint (4001)
|
|
575
|
-
// data (b1[] 0 -> 1)
|
|
576
|
-
// partial_checkpoint_complete (4001, priority 1)
|
|
577
|
-
// data (b0a[], 0 -> 2000)
|
|
578
|
-
// ## adds new data, interrupting the checkpoint
|
|
579
|
-
// data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
|
|
580
|
-
// checkpoint_diff (4004)
|
|
581
|
-
// data (b1[], 1 -> 4002)
|
|
582
|
-
// partial_checkpoint_complete (4004, priority 1)
|
|
583
|
-
// data (b0a[], 4000 -> 4003)
|
|
584
|
-
// data (b0b[], 0 -> 1999)
|
|
585
|
-
// data (b0b[], 1999 -> 3999)
|
|
586
|
-
// data (b0b[], 3999 -> 4004)
|
|
587
|
-
// checkpoint_complete (4004)
|
|
588
|
-
expect(lines).toMatchSnapshot();
|
|
589
|
-
expect(sentRows).toBe(4004);
|
|
590
|
-
}
|
|
591
|
-
catch (e_5) {
|
|
592
|
-
env_5.error = e_5;
|
|
593
|
-
env_5.hasError = true;
|
|
594
|
-
}
|
|
595
|
-
finally {
|
|
596
|
-
const result_5 = __disposeResources(env_5);
|
|
597
|
-
if (result_5)
|
|
598
|
-
await result_5;
|
|
599
535
|
}
|
|
536
|
+
// Expected lines (full details in snapshot):
|
|
537
|
+
//
|
|
538
|
+
// checkpoint (4001)
|
|
539
|
+
// data (b1[] 0 -> 1)
|
|
540
|
+
// partial_checkpoint_complete (4001, priority 1)
|
|
541
|
+
// data (b0a[], 0 -> 2000)
|
|
542
|
+
// ## adds new data, interrupting the checkpoint
|
|
543
|
+
// data (b0a[], 2000 -> 4000) # expected - stream is already busy with this by the time it receives the interruption
|
|
544
|
+
// checkpoint_diff (4004)
|
|
545
|
+
// data (b1[], 1 -> 4002)
|
|
546
|
+
// partial_checkpoint_complete (4004, priority 1)
|
|
547
|
+
// data (b0a[], 4000 -> 4003)
|
|
548
|
+
// data (b0b[], 0 -> 1999)
|
|
549
|
+
// data (b0b[], 1999 -> 3999)
|
|
550
|
+
// data (b0b[], 3999 -> 4004)
|
|
551
|
+
// checkpoint_complete (4004)
|
|
552
|
+
expect(lines).toMatchSnapshot();
|
|
553
|
+
expect(sentRows).toBe(4004);
|
|
600
554
|
});
|
|
601
555
|
test('sends checkpoint complete line for empty checkpoint', async () => {
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
if (
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
else {
|
|
651
|
-
break;
|
|
652
|
-
}
|
|
556
|
+
await using f = await factory();
|
|
557
|
+
const syncRules = await updateSyncRules(f, {
|
|
558
|
+
content: BASIC_SYNC_RULES
|
|
559
|
+
});
|
|
560
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
561
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
562
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
563
|
+
await writer.markAllSnapshotDone('0/1');
|
|
564
|
+
await writer.save({
|
|
565
|
+
sourceTable: testTable,
|
|
566
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
567
|
+
after: {
|
|
568
|
+
id: 't1',
|
|
569
|
+
description: 'sync'
|
|
570
|
+
},
|
|
571
|
+
afterReplicaId: 't1'
|
|
572
|
+
});
|
|
573
|
+
await writer.commit('0/1');
|
|
574
|
+
const stream = sync.streamResponse({
|
|
575
|
+
syncContext,
|
|
576
|
+
bucketStorage,
|
|
577
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
578
|
+
params: {
|
|
579
|
+
buckets: [],
|
|
580
|
+
include_checksum: true,
|
|
581
|
+
raw_data: true
|
|
582
|
+
},
|
|
583
|
+
tracker,
|
|
584
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 100000 }),
|
|
585
|
+
isEncodingAsBson: false
|
|
586
|
+
});
|
|
587
|
+
const lines = [];
|
|
588
|
+
let receivedCompletions = 0;
|
|
589
|
+
for await (let next of stream) {
|
|
590
|
+
if (typeof next == 'string') {
|
|
591
|
+
next = JSON.parse(next);
|
|
592
|
+
}
|
|
593
|
+
lines.push(next);
|
|
594
|
+
if (typeof next === 'object' && next !== null) {
|
|
595
|
+
if ('checkpoint_complete' in next) {
|
|
596
|
+
receivedCompletions++;
|
|
597
|
+
if (receivedCompletions == 1) {
|
|
598
|
+
// Trigger an empty bucket update.
|
|
599
|
+
await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } });
|
|
600
|
+
await writer.commit('1/0');
|
|
601
|
+
}
|
|
602
|
+
else {
|
|
603
|
+
break;
|
|
653
604
|
}
|
|
654
605
|
}
|
|
655
606
|
}
|
|
656
|
-
expect(lines).toMatchSnapshot();
|
|
657
|
-
}
|
|
658
|
-
catch (e_6) {
|
|
659
|
-
env_6.error = e_6;
|
|
660
|
-
env_6.hasError = true;
|
|
661
|
-
}
|
|
662
|
-
finally {
|
|
663
|
-
const result_6 = __disposeResources(env_6);
|
|
664
|
-
if (result_6)
|
|
665
|
-
await result_6;
|
|
666
607
|
}
|
|
608
|
+
expect(lines).toMatchSnapshot();
|
|
667
609
|
});
|
|
668
610
|
test('sync legacy non-raw data', async () => {
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
// Specifically check the number - this is the important part of the test
|
|
706
|
-
expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
|
|
707
|
-
}
|
|
708
|
-
catch (e_7) {
|
|
709
|
-
env_7.error = e_7;
|
|
710
|
-
env_7.hasError = true;
|
|
711
|
-
}
|
|
712
|
-
finally {
|
|
713
|
-
const result_7 = __disposeResources(env_7);
|
|
714
|
-
if (result_7)
|
|
715
|
-
await result_7;
|
|
716
|
-
}
|
|
611
|
+
await using f = await factory();
|
|
612
|
+
const syncRules = await updateSyncRules(f, {
|
|
613
|
+
content: BASIC_SYNC_RULES
|
|
614
|
+
});
|
|
615
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
616
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
617
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
618
|
+
await writer.markAllSnapshotDone('0/1');
|
|
619
|
+
await writer.save({
|
|
620
|
+
sourceTable: testTable,
|
|
621
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
622
|
+
after: {
|
|
623
|
+
id: 't1',
|
|
624
|
+
description: 'Test\n"string"',
|
|
625
|
+
large_num: 12345678901234567890n
|
|
626
|
+
},
|
|
627
|
+
afterReplicaId: 't1'
|
|
628
|
+
});
|
|
629
|
+
await writer.commit('0/1');
|
|
630
|
+
const stream = sync.streamResponse({
|
|
631
|
+
syncContext,
|
|
632
|
+
bucketStorage,
|
|
633
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
634
|
+
params: {
|
|
635
|
+
buckets: [],
|
|
636
|
+
include_checksum: true,
|
|
637
|
+
raw_data: false
|
|
638
|
+
},
|
|
639
|
+
tracker,
|
|
640
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
641
|
+
isEncodingAsBson: false
|
|
642
|
+
});
|
|
643
|
+
const lines = await consumeCheckpointLines(stream);
|
|
644
|
+
expect(lines).toMatchSnapshot();
|
|
645
|
+
// Specifically check the number - this is the important part of the test
|
|
646
|
+
expect(lines[1].data.data[0].data.large_num).toEqual(12345678901234567890n);
|
|
717
647
|
});
|
|
718
648
|
test('expired token', async () => {
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
const lines = await consumeCheckpointLines(stream);
|
|
740
|
-
expect(lines).toMatchSnapshot();
|
|
741
|
-
}
|
|
742
|
-
catch (e_8) {
|
|
743
|
-
env_8.error = e_8;
|
|
744
|
-
env_8.hasError = true;
|
|
745
|
-
}
|
|
746
|
-
finally {
|
|
747
|
-
const result_8 = __disposeResources(env_8);
|
|
748
|
-
if (result_8)
|
|
749
|
-
await result_8;
|
|
750
|
-
}
|
|
649
|
+
await using f = await factory();
|
|
650
|
+
const syncRules = await updateSyncRules(f, {
|
|
651
|
+
content: BASIC_SYNC_RULES
|
|
652
|
+
});
|
|
653
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
654
|
+
const stream = sync.streamResponse({
|
|
655
|
+
syncContext,
|
|
656
|
+
bucketStorage,
|
|
657
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
658
|
+
params: {
|
|
659
|
+
buckets: [],
|
|
660
|
+
include_checksum: true,
|
|
661
|
+
raw_data: true
|
|
662
|
+
},
|
|
663
|
+
tracker,
|
|
664
|
+
token: new JwtPayload({ sub: '', exp: 0 }),
|
|
665
|
+
isEncodingAsBson: false
|
|
666
|
+
});
|
|
667
|
+
const lines = await consumeCheckpointLines(stream);
|
|
668
|
+
expect(lines).toMatchSnapshot();
|
|
751
669
|
});
|
|
752
670
|
test('sync updates to global data', async (context) => {
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
await writer.commit('0/2');
|
|
804
|
-
expect(await getCheckpointLines(iter)).toMatchSnapshot();
|
|
805
|
-
}
|
|
806
|
-
catch (e_9) {
|
|
807
|
-
env_9.error = e_9;
|
|
808
|
-
env_9.hasError = true;
|
|
809
|
-
}
|
|
810
|
-
finally {
|
|
811
|
-
const result_9 = __disposeResources(env_9);
|
|
812
|
-
if (result_9)
|
|
813
|
-
await result_9;
|
|
814
|
-
}
|
|
671
|
+
await using f = await factory();
|
|
672
|
+
const syncRules = await updateSyncRules(f, {
|
|
673
|
+
content: BASIC_SYNC_RULES
|
|
674
|
+
});
|
|
675
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
676
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
677
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
678
|
+
// Activate
|
|
679
|
+
await writer.markAllSnapshotDone('0/0');
|
|
680
|
+
await writer.keepalive('0/0');
|
|
681
|
+
const stream = sync.streamResponse({
|
|
682
|
+
syncContext,
|
|
683
|
+
bucketStorage,
|
|
684
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
685
|
+
params: {
|
|
686
|
+
buckets: [],
|
|
687
|
+
include_checksum: true,
|
|
688
|
+
raw_data: true
|
|
689
|
+
},
|
|
690
|
+
tracker,
|
|
691
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
692
|
+
isEncodingAsBson: false
|
|
693
|
+
});
|
|
694
|
+
const iter = stream[Symbol.asyncIterator]();
|
|
695
|
+
context.onTestFinished(() => {
|
|
696
|
+
iter.return?.();
|
|
697
|
+
});
|
|
698
|
+
expect(await getCheckpointLines(iter)).toMatchSnapshot();
|
|
699
|
+
await writer.save({
|
|
700
|
+
sourceTable: testTable,
|
|
701
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
702
|
+
after: {
|
|
703
|
+
id: 't1',
|
|
704
|
+
description: 'Test 1'
|
|
705
|
+
},
|
|
706
|
+
afterReplicaId: 't1'
|
|
707
|
+
});
|
|
708
|
+
await writer.commit('0/1');
|
|
709
|
+
expect(await getCheckpointLines(iter)).toMatchSnapshot();
|
|
710
|
+
await writer.save({
|
|
711
|
+
sourceTable: testTable,
|
|
712
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
713
|
+
after: {
|
|
714
|
+
id: 't2',
|
|
715
|
+
description: 'Test 2'
|
|
716
|
+
},
|
|
717
|
+
afterReplicaId: 't2'
|
|
718
|
+
});
|
|
719
|
+
await writer.commit('0/2');
|
|
720
|
+
expect(await getCheckpointLines(iter)).toMatchSnapshot();
|
|
815
721
|
});
|
|
816
722
|
test('sync updates to parameter query only', async (context) => {
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
const syncRules = await updateSyncRules(f, {
|
|
821
|
-
content: `bucket_definitions:
|
|
723
|
+
await using f = await factory();
|
|
724
|
+
const syncRules = await updateSyncRules(f, {
|
|
725
|
+
content: `bucket_definitions:
|
|
822
726
|
by_user:
|
|
823
727
|
parameters: select users.id as user_id from users where users.id = request.user_id()
|
|
824
728
|
data:
|
|
825
729
|
- select * from lists where user_id = bucket.user_id
|
|
826
730
|
`
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
}
|
|
871
|
-
catch (e_10) {
|
|
872
|
-
env_10.error = e_10;
|
|
873
|
-
env_10.hasError = true;
|
|
874
|
-
}
|
|
875
|
-
finally {
|
|
876
|
-
const result_10 = __disposeResources(env_10);
|
|
877
|
-
if (result_10)
|
|
878
|
-
await result_10;
|
|
879
|
-
}
|
|
731
|
+
});
|
|
732
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
733
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
734
|
+
const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
|
|
735
|
+
// Activate
|
|
736
|
+
await writer.markAllSnapshotDone('0/0');
|
|
737
|
+
await writer.keepalive('0/0');
|
|
738
|
+
const stream = sync.streamResponse({
|
|
739
|
+
syncContext,
|
|
740
|
+
bucketStorage,
|
|
741
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
742
|
+
params: {
|
|
743
|
+
buckets: [],
|
|
744
|
+
include_checksum: true,
|
|
745
|
+
raw_data: true
|
|
746
|
+
},
|
|
747
|
+
tracker,
|
|
748
|
+
token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
|
|
749
|
+
isEncodingAsBson: false
|
|
750
|
+
});
|
|
751
|
+
const iter = stream[Symbol.asyncIterator]();
|
|
752
|
+
context.onTestFinished(() => {
|
|
753
|
+
iter.return?.();
|
|
754
|
+
});
|
|
755
|
+
// Initial empty checkpoint
|
|
756
|
+
const checkpoint1 = await getCheckpointLines(iter);
|
|
757
|
+
expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([]);
|
|
758
|
+
expect(checkpoint1).toMatchSnapshot();
|
|
759
|
+
// Add user
|
|
760
|
+
await writer.save({
|
|
761
|
+
sourceTable: usersTable,
|
|
762
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
763
|
+
after: {
|
|
764
|
+
id: 'user1',
|
|
765
|
+
name: 'User 1'
|
|
766
|
+
},
|
|
767
|
+
afterReplicaId: 'user1'
|
|
768
|
+
});
|
|
769
|
+
await writer.commit('0/1');
|
|
770
|
+
const checkpoint2 = await getCheckpointLines(iter);
|
|
771
|
+
const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
|
|
772
|
+
expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
|
|
773
|
+
expect(checkpoint2).toMatchSnapshot();
|
|
880
774
|
});
|
|
881
775
|
test('sync updates to data query only', async (context) => {
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
const syncRules = await updateSyncRules(f, {
|
|
886
|
-
content: `bucket_definitions:
|
|
776
|
+
await using f = await factory();
|
|
777
|
+
const syncRules = await updateSyncRules(f, {
|
|
778
|
+
content: `bucket_definitions:
|
|
887
779
|
by_user:
|
|
888
780
|
parameters: select users.id as user_id from users where users.id = request.user_id()
|
|
889
781
|
data:
|
|
890
782
|
- select * from lists where user_id = bucket.user_id
|
|
891
783
|
`
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
}
|
|
944
|
-
catch (e_11) {
|
|
945
|
-
env_11.error = e_11;
|
|
946
|
-
env_11.hasError = true;
|
|
947
|
-
}
|
|
948
|
-
finally {
|
|
949
|
-
const result_11 = __disposeResources(env_11);
|
|
950
|
-
if (result_11)
|
|
951
|
-
await result_11;
|
|
952
|
-
}
|
|
784
|
+
});
|
|
785
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
786
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
787
|
+
const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
|
|
788
|
+
const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
|
|
789
|
+
await writer.markAllSnapshotDone('0/1');
|
|
790
|
+
await writer.save({
|
|
791
|
+
sourceTable: usersTable,
|
|
792
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
793
|
+
after: {
|
|
794
|
+
id: 'user1',
|
|
795
|
+
name: 'User 1'
|
|
796
|
+
},
|
|
797
|
+
afterReplicaId: 'user1'
|
|
798
|
+
});
|
|
799
|
+
await writer.commit('0/1');
|
|
800
|
+
const stream = sync.streamResponse({
|
|
801
|
+
syncContext,
|
|
802
|
+
bucketStorage,
|
|
803
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
804
|
+
params: {
|
|
805
|
+
buckets: [],
|
|
806
|
+
include_checksum: true,
|
|
807
|
+
raw_data: true
|
|
808
|
+
},
|
|
809
|
+
tracker,
|
|
810
|
+
token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
|
|
811
|
+
isEncodingAsBson: false
|
|
812
|
+
});
|
|
813
|
+
const iter = stream[Symbol.asyncIterator]();
|
|
814
|
+
context.onTestFinished(() => {
|
|
815
|
+
iter.return?.();
|
|
816
|
+
});
|
|
817
|
+
const { bucket } = bucketRequest(syncRules, 'by_user["user1"]');
|
|
818
|
+
const checkpoint1 = await getCheckpointLines(iter);
|
|
819
|
+
expect(checkpoint1[0].checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]);
|
|
820
|
+
expect(checkpoint1).toMatchSnapshot();
|
|
821
|
+
await writer.save({
|
|
822
|
+
sourceTable: listsTable,
|
|
823
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
824
|
+
after: {
|
|
825
|
+
id: 'list1',
|
|
826
|
+
user_id: 'user1',
|
|
827
|
+
name: 'User 1'
|
|
828
|
+
},
|
|
829
|
+
afterReplicaId: 'list1'
|
|
830
|
+
});
|
|
831
|
+
await writer.commit('0/1');
|
|
832
|
+
const checkpoint2 = await getCheckpointLines(iter);
|
|
833
|
+
expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
|
|
834
|
+
expect(checkpoint2).toMatchSnapshot();
|
|
953
835
|
});
|
|
954
836
|
test('sync updates to parameter query + data', async (context) => {
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
const syncRules = await updateSyncRules(f, {
|
|
959
|
-
content: `bucket_definitions:
|
|
837
|
+
await using f = await factory();
|
|
838
|
+
const syncRules = await updateSyncRules(f, {
|
|
839
|
+
content: `bucket_definitions:
|
|
960
840
|
by_user:
|
|
961
841
|
parameters: select users.id as user_id from users where users.id = request.user_id()
|
|
962
842
|
data:
|
|
963
843
|
- select * from lists where user_id = bucket.user_id
|
|
964
844
|
`
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
}
|
|
1018
|
-
catch (e_12) {
|
|
1019
|
-
env_12.error = e_12;
|
|
1020
|
-
env_12.hasError = true;
|
|
1021
|
-
}
|
|
1022
|
-
finally {
|
|
1023
|
-
const result_12 = __disposeResources(env_12);
|
|
1024
|
-
if (result_12)
|
|
1025
|
-
await result_12;
|
|
1026
|
-
}
|
|
845
|
+
});
|
|
846
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
847
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
848
|
+
const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1);
|
|
849
|
+
const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2);
|
|
850
|
+
// Activate
|
|
851
|
+
await writer.markAllSnapshotDone('0/0');
|
|
852
|
+
await writer.keepalive('0/0');
|
|
853
|
+
const stream = sync.streamResponse({
|
|
854
|
+
syncContext,
|
|
855
|
+
bucketStorage,
|
|
856
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
857
|
+
params: {
|
|
858
|
+
buckets: [],
|
|
859
|
+
include_checksum: true,
|
|
860
|
+
raw_data: true
|
|
861
|
+
},
|
|
862
|
+
tracker,
|
|
863
|
+
token: new JwtPayload({ sub: 'user1', exp: Date.now() / 1000 + 100 }),
|
|
864
|
+
isEncodingAsBson: false
|
|
865
|
+
});
|
|
866
|
+
const iter = stream[Symbol.asyncIterator]();
|
|
867
|
+
context.onTestFinished(() => {
|
|
868
|
+
iter.return?.();
|
|
869
|
+
});
|
|
870
|
+
// Initial empty checkpoint
|
|
871
|
+
expect(await getCheckpointLines(iter)).toMatchSnapshot();
|
|
872
|
+
await writer.markAllSnapshotDone('0/1');
|
|
873
|
+
await writer.save({
|
|
874
|
+
sourceTable: listsTable,
|
|
875
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
876
|
+
after: {
|
|
877
|
+
id: 'list1',
|
|
878
|
+
user_id: 'user1',
|
|
879
|
+
name: 'User 1'
|
|
880
|
+
},
|
|
881
|
+
afterReplicaId: 'list1'
|
|
882
|
+
});
|
|
883
|
+
await writer.save({
|
|
884
|
+
sourceTable: usersTable,
|
|
885
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
886
|
+
after: {
|
|
887
|
+
id: 'user1',
|
|
888
|
+
name: 'User 1'
|
|
889
|
+
},
|
|
890
|
+
afterReplicaId: 'user1'
|
|
891
|
+
});
|
|
892
|
+
await writer.commit('0/1');
|
|
893
|
+
const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]');
|
|
894
|
+
const checkpoint2 = await getCheckpointLines(iter);
|
|
895
|
+
expect(checkpoint2[0].checkpoint_diff?.updated_buckets?.map((b) => b.bucket)).toEqual([bucket]);
|
|
896
|
+
expect(checkpoint2).toMatchSnapshot();
|
|
1027
897
|
});
|
|
1028
898
|
test('expiring token', async (context) => {
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
const expLines = await getCheckpointLines(iter);
|
|
1061
|
-
expect(expLines).toMatchSnapshot();
|
|
1062
|
-
}
|
|
1063
|
-
catch (e_13) {
|
|
1064
|
-
env_13.error = e_13;
|
|
1065
|
-
env_13.hasError = true;
|
|
1066
|
-
}
|
|
1067
|
-
finally {
|
|
1068
|
-
const result_13 = __disposeResources(env_13);
|
|
1069
|
-
if (result_13)
|
|
1070
|
-
await result_13;
|
|
1071
|
-
}
|
|
899
|
+
await using f = await factory();
|
|
900
|
+
const syncRules = await updateSyncRules(f, {
|
|
901
|
+
content: BASIC_SYNC_RULES
|
|
902
|
+
});
|
|
903
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
904
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
905
|
+
// Activate
|
|
906
|
+
await writer.markAllSnapshotDone('0/0');
|
|
907
|
+
await writer.keepalive('0/0');
|
|
908
|
+
const exp = Date.now() / 1000 + 0.1;
|
|
909
|
+
const stream = sync.streamResponse({
|
|
910
|
+
syncContext,
|
|
911
|
+
bucketStorage,
|
|
912
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
913
|
+
params: {
|
|
914
|
+
buckets: [],
|
|
915
|
+
include_checksum: true,
|
|
916
|
+
raw_data: true
|
|
917
|
+
},
|
|
918
|
+
tracker,
|
|
919
|
+
token: new JwtPayload({ sub: '', exp: exp }),
|
|
920
|
+
isEncodingAsBson: false
|
|
921
|
+
});
|
|
922
|
+
const iter = stream[Symbol.asyncIterator]();
|
|
923
|
+
context.onTestFinished(() => {
|
|
924
|
+
iter.return?.();
|
|
925
|
+
});
|
|
926
|
+
const checkpoint = await getCheckpointLines(iter);
|
|
927
|
+
expect(checkpoint).toMatchSnapshot();
|
|
928
|
+
const expLines = await getCheckpointLines(iter);
|
|
929
|
+
expect(expLines).toMatchSnapshot();
|
|
1072
930
|
});
|
|
1073
931
|
test('compacting data - invalidate checkpoint', async (context) => {
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
932
|
+
// This tests a case of a compact operation invalidating a checkpoint in the
|
|
933
|
+
// middle of syncing data.
|
|
934
|
+
// This is expected to be rare in practice, but it is important to handle
|
|
935
|
+
// this case correctly to maintain consistency on the client.
|
|
936
|
+
await using f = await factory();
|
|
937
|
+
const syncRules = await updateSyncRules(f, {
|
|
938
|
+
content: BASIC_SYNC_RULES
|
|
939
|
+
});
|
|
940
|
+
const bucketStorage = await f.getInstance(syncRules);
|
|
941
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
942
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
943
|
+
await writer.markAllSnapshotDone('0/1');
|
|
944
|
+
await writer.save({
|
|
945
|
+
sourceTable: testTable,
|
|
946
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
947
|
+
after: {
|
|
948
|
+
id: 't1',
|
|
949
|
+
description: 'Test 1'
|
|
950
|
+
},
|
|
951
|
+
afterReplicaId: 't1'
|
|
952
|
+
});
|
|
953
|
+
await writer.save({
|
|
954
|
+
sourceTable: testTable,
|
|
955
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
956
|
+
after: {
|
|
957
|
+
id: 't2',
|
|
958
|
+
description: 'Test 2'
|
|
959
|
+
},
|
|
960
|
+
afterReplicaId: 't2'
|
|
961
|
+
});
|
|
962
|
+
await writer.commit('0/1');
|
|
963
|
+
const stream = sync.streamResponse({
|
|
964
|
+
syncContext,
|
|
965
|
+
bucketStorage,
|
|
966
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
967
|
+
params: {
|
|
968
|
+
buckets: [],
|
|
969
|
+
include_checksum: true,
|
|
970
|
+
raw_data: true
|
|
971
|
+
},
|
|
972
|
+
tracker,
|
|
973
|
+
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
974
|
+
isEncodingAsBson: false
|
|
975
|
+
});
|
|
976
|
+
const iter = stream[Symbol.asyncIterator]();
|
|
977
|
+
context.onTestFinished(() => {
|
|
978
|
+
iter.return?.();
|
|
979
|
+
});
|
|
980
|
+
// Only consume the first "checkpoint" message, and pause before receiving data.
|
|
981
|
+
const lines = await consumeIterator(iter, { consume: false, isDone: (line) => line?.checkpoint != null });
|
|
982
|
+
expect(lines).toMatchSnapshot();
|
|
983
|
+
expect(lines[0]).toEqual({
|
|
984
|
+
checkpoint: expect.objectContaining({
|
|
985
|
+
last_op_id: '2'
|
|
986
|
+
})
|
|
987
|
+
});
|
|
988
|
+
// Now we save additional data AND compact before continuing.
|
|
989
|
+
// This invalidates the checkpoint we've received above.
|
|
990
|
+
await writer.markAllSnapshotDone('0/1');
|
|
991
|
+
await writer.save({
|
|
992
|
+
sourceTable: testTable,
|
|
993
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
994
|
+
after: {
|
|
995
|
+
id: 't1',
|
|
996
|
+
description: 'Test 1b'
|
|
997
|
+
},
|
|
998
|
+
afterReplicaId: 't1'
|
|
999
|
+
});
|
|
1000
|
+
await writer.save({
|
|
1001
|
+
sourceTable: testTable,
|
|
1002
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
1003
|
+
after: {
|
|
1004
|
+
id: 't2',
|
|
1005
|
+
description: 'Test 2b'
|
|
1006
|
+
},
|
|
1007
|
+
afterReplicaId: 't2'
|
|
1008
|
+
});
|
|
1009
|
+
await writer.commit('0/2');
|
|
1010
|
+
await bucketStorage.compact({
|
|
1011
|
+
minBucketChanges: 1,
|
|
1012
|
+
minChangeRatio: 0
|
|
1013
|
+
});
|
|
1014
|
+
const lines2 = await getCheckpointLines(iter, { consume: true });
|
|
1015
|
+
// Snapshot test checks for changes in general.
|
|
1016
|
+
// The tests after that documents the specific things we're looking for
|
|
1017
|
+
// in this test.
|
|
1018
|
+
expect(lines2).toMatchSnapshot();
|
|
1019
|
+
expect(lines2[0]).toEqual({
|
|
1020
|
+
data: expect.objectContaining({
|
|
1021
|
+
has_more: false,
|
|
1022
|
+
data: [
|
|
1023
|
+
// The first two ops have been replaced by a single CLEAR op
|
|
1024
|
+
expect.objectContaining({
|
|
1025
|
+
op: 'CLEAR'
|
|
1026
|
+
})
|
|
1027
|
+
]
|
|
1028
|
+
})
|
|
1029
|
+
});
|
|
1030
|
+
// Note: No checkpoint_complete here, since the checkpoint has been
|
|
1031
|
+
// invalidated by the CLEAR op.
|
|
1032
|
+
expect(lines2[1]).toEqual({
|
|
1033
|
+
checkpoint_diff: expect.objectContaining({
|
|
1034
|
+
last_op_id: '4'
|
|
1035
|
+
})
|
|
1036
|
+
});
|
|
1037
|
+
expect(lines2[2]).toEqual({
|
|
1038
|
+
data: expect.objectContaining({
|
|
1039
|
+
has_more: false,
|
|
1040
|
+
data: [
|
|
1041
|
+
expect.objectContaining({
|
|
1042
|
+
op: 'PUT'
|
|
1043
|
+
}),
|
|
1044
|
+
expect.objectContaining({
|
|
1045
|
+
op: 'PUT'
|
|
1046
|
+
})
|
|
1047
|
+
]
|
|
1048
|
+
})
|
|
1049
|
+
});
|
|
1050
|
+
// Now we get a checkpoint_complete
|
|
1051
|
+
expect(lines2[3]).toEqual({
|
|
1052
|
+
checkpoint_complete: expect.objectContaining({
|
|
1053
|
+
last_op_id: '4'
|
|
1054
|
+
})
|
|
1055
|
+
});
|
|
1056
|
+
});
|
|
1057
|
+
test('write checkpoint', async () => {
|
|
1058
|
+
await using f = await factory();
|
|
1059
|
+
const syncRules = await updateSyncRules(f, {
|
|
1060
|
+
content: BASIC_SYNC_RULES
|
|
1061
|
+
});
|
|
1062
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
1063
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1064
|
+
await writer.markAllSnapshotDone('0/1');
|
|
1065
|
+
// <= the managed write checkpoint LSN below
|
|
1066
|
+
await writer.commit('0/1');
|
|
1067
|
+
const checkpoint = await bucketStorage.createManagedWriteCheckpoint({
|
|
1068
|
+
user_id: 'test',
|
|
1069
|
+
heads: { '1': '1/0' }
|
|
1070
|
+
});
|
|
1071
|
+
const params = {
|
|
1072
|
+
syncContext,
|
|
1073
|
+
bucketStorage,
|
|
1074
|
+
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
1075
|
+
params: {
|
|
1076
|
+
buckets: [],
|
|
1077
|
+
include_checksum: true,
|
|
1078
|
+
raw_data: true
|
|
1079
|
+
},
|
|
1080
|
+
tracker,
|
|
1081
|
+
token: new JwtPayload({ sub: 'test', exp: Date.now() / 1000 + 10 }),
|
|
1082
|
+
isEncodingAsBson: false
|
|
1083
|
+
};
|
|
1084
|
+
const stream1 = sync.streamResponse(params);
|
|
1085
|
+
const lines1 = await consumeCheckpointLines(stream1);
|
|
1086
|
+
// If write checkpoints are not correctly filtered, this may already
|
|
1087
|
+
// contain the write checkpoint.
|
|
1088
|
+
expect(lines1[0]).toMatchObject({
|
|
1089
|
+
checkpoint: expect.objectContaining({
|
|
1090
|
+
last_op_id: '0',
|
|
1091
|
+
write_checkpoint: undefined
|
|
1092
|
+
})
|
|
1093
|
+
});
|
|
1094
|
+
await writer.markAllSnapshotDone('0/1');
|
|
1095
|
+
// must be >= the managed write checkpoint LSN
|
|
1096
|
+
await writer.commit('1/0');
|
|
1097
|
+
// At this point the LSN has advanced, so the write checkpoint should be
|
|
1098
|
+
// included in the next checkpoint message.
|
|
1099
|
+
const stream2 = sync.streamResponse(params);
|
|
1100
|
+
const lines2 = await consumeCheckpointLines(stream2);
|
|
1101
|
+
expect(lines2[0]).toMatchObject({
|
|
1102
|
+
checkpoint: expect.objectContaining({
|
|
1103
|
+
last_op_id: '0',
|
|
1104
|
+
write_checkpoint: `${checkpoint}`
|
|
1105
|
+
})
|
|
1106
|
+
});
|
|
1107
|
+
});
|
|
1108
|
+
test('encodes sync rules id in buckets for streams', async () => {
|
|
1109
|
+
await using f = await factory();
|
|
1110
|
+
// This test relies making an actual update to sync rules to test the different bucket names.
|
|
1111
|
+
// The actual naming scheme may change, as long as the two buckets have different names.
|
|
1112
|
+
const rules = [
|
|
1113
|
+
`
|
|
1114
|
+
streams:
|
|
1115
|
+
test:
|
|
1116
|
+
auto_subscribe: true
|
|
1117
|
+
query: SELECT * FROM test;
|
|
1118
|
+
|
|
1119
|
+
config:
|
|
1120
|
+
edition: 2
|
|
1121
|
+
`,
|
|
1122
|
+
`
|
|
1123
|
+
streams:
|
|
1124
|
+
test2:
|
|
1125
|
+
auto_subscribe: true
|
|
1126
|
+
query: SELECT * FROM test WHERE 1;
|
|
1127
|
+
|
|
1128
|
+
config:
|
|
1129
|
+
edition: 2
|
|
1130
|
+
`
|
|
1131
|
+
];
|
|
1132
|
+
for (let i = 0; i < 2; i++) {
|
|
1081
1133
|
const syncRules = await updateSyncRules(f, {
|
|
1082
|
-
content:
|
|
1134
|
+
content: rules[i]
|
|
1083
1135
|
});
|
|
1084
|
-
const bucketStorage =
|
|
1085
|
-
|
|
1086
|
-
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1136
|
+
const bucketStorage = f.getInstance(syncRules);
|
|
1137
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1138
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1);
|
|
1087
1139
|
await writer.markAllSnapshotDone('0/1');
|
|
1088
1140
|
await writer.save({
|
|
1089
1141
|
sourceTable: testTable,
|
|
@@ -1094,19 +1146,10 @@ bucket_definitions:
|
|
|
1094
1146
|
},
|
|
1095
1147
|
afterReplicaId: 't1'
|
|
1096
1148
|
});
|
|
1097
|
-
await writer.save({
|
|
1098
|
-
sourceTable: testTable,
|
|
1099
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1100
|
-
after: {
|
|
1101
|
-
id: 't2',
|
|
1102
|
-
description: 'Test 2'
|
|
1103
|
-
},
|
|
1104
|
-
afterReplicaId: 't2'
|
|
1105
|
-
});
|
|
1106
1149
|
await writer.commit('0/1');
|
|
1107
1150
|
const stream = sync.streamResponse({
|
|
1108
1151
|
syncContext,
|
|
1109
|
-
bucketStorage,
|
|
1152
|
+
bucketStorage: bucketStorage,
|
|
1110
1153
|
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
1111
1154
|
params: {
|
|
1112
1155
|
buckets: [],
|
|
@@ -1117,241 +1160,8 @@ bucket_definitions:
|
|
|
1117
1160
|
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
1118
1161
|
isEncodingAsBson: false
|
|
1119
1162
|
});
|
|
1120
|
-
const
|
|
1121
|
-
context.onTestFinished(() => {
|
|
1122
|
-
iter.return?.();
|
|
1123
|
-
});
|
|
1124
|
-
// Only consume the first "checkpoint" message, and pause before receiving data.
|
|
1125
|
-
const lines = await consumeIterator(iter, { consume: false, isDone: (line) => line?.checkpoint != null });
|
|
1163
|
+
const lines = await consumeCheckpointLines(stream);
|
|
1126
1164
|
expect(lines).toMatchSnapshot();
|
|
1127
|
-
expect(lines[0]).toEqual({
|
|
1128
|
-
checkpoint: expect.objectContaining({
|
|
1129
|
-
last_op_id: '2'
|
|
1130
|
-
})
|
|
1131
|
-
});
|
|
1132
|
-
// Now we save additional data AND compact before continuing.
|
|
1133
|
-
// This invalidates the checkpoint we've received above.
|
|
1134
|
-
await writer.markAllSnapshotDone('0/1');
|
|
1135
|
-
await writer.save({
|
|
1136
|
-
sourceTable: testTable,
|
|
1137
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
1138
|
-
after: {
|
|
1139
|
-
id: 't1',
|
|
1140
|
-
description: 'Test 1b'
|
|
1141
|
-
},
|
|
1142
|
-
afterReplicaId: 't1'
|
|
1143
|
-
});
|
|
1144
|
-
await writer.save({
|
|
1145
|
-
sourceTable: testTable,
|
|
1146
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
1147
|
-
after: {
|
|
1148
|
-
id: 't2',
|
|
1149
|
-
description: 'Test 2b'
|
|
1150
|
-
},
|
|
1151
|
-
afterReplicaId: 't2'
|
|
1152
|
-
});
|
|
1153
|
-
await writer.commit('0/2');
|
|
1154
|
-
await bucketStorage.compact({
|
|
1155
|
-
minBucketChanges: 1,
|
|
1156
|
-
minChangeRatio: 0
|
|
1157
|
-
});
|
|
1158
|
-
const lines2 = await getCheckpointLines(iter, { consume: true });
|
|
1159
|
-
// Snapshot test checks for changes in general.
|
|
1160
|
-
// The tests after that documents the specific things we're looking for
|
|
1161
|
-
// in this test.
|
|
1162
|
-
expect(lines2).toMatchSnapshot();
|
|
1163
|
-
expect(lines2[0]).toEqual({
|
|
1164
|
-
data: expect.objectContaining({
|
|
1165
|
-
has_more: false,
|
|
1166
|
-
data: [
|
|
1167
|
-
// The first two ops have been replaced by a single CLEAR op
|
|
1168
|
-
expect.objectContaining({
|
|
1169
|
-
op: 'CLEAR'
|
|
1170
|
-
})
|
|
1171
|
-
]
|
|
1172
|
-
})
|
|
1173
|
-
});
|
|
1174
|
-
// Note: No checkpoint_complete here, since the checkpoint has been
|
|
1175
|
-
// invalidated by the CLEAR op.
|
|
1176
|
-
expect(lines2[1]).toEqual({
|
|
1177
|
-
checkpoint_diff: expect.objectContaining({
|
|
1178
|
-
last_op_id: '4'
|
|
1179
|
-
})
|
|
1180
|
-
});
|
|
1181
|
-
expect(lines2[2]).toEqual({
|
|
1182
|
-
data: expect.objectContaining({
|
|
1183
|
-
has_more: false,
|
|
1184
|
-
data: [
|
|
1185
|
-
expect.objectContaining({
|
|
1186
|
-
op: 'PUT'
|
|
1187
|
-
}),
|
|
1188
|
-
expect.objectContaining({
|
|
1189
|
-
op: 'PUT'
|
|
1190
|
-
})
|
|
1191
|
-
]
|
|
1192
|
-
})
|
|
1193
|
-
});
|
|
1194
|
-
// Now we get a checkpoint_complete
|
|
1195
|
-
expect(lines2[3]).toEqual({
|
|
1196
|
-
checkpoint_complete: expect.objectContaining({
|
|
1197
|
-
last_op_id: '4'
|
|
1198
|
-
})
|
|
1199
|
-
});
|
|
1200
|
-
}
|
|
1201
|
-
catch (e_14) {
|
|
1202
|
-
env_14.error = e_14;
|
|
1203
|
-
env_14.hasError = true;
|
|
1204
|
-
}
|
|
1205
|
-
finally {
|
|
1206
|
-
const result_14 = __disposeResources(env_14);
|
|
1207
|
-
if (result_14)
|
|
1208
|
-
await result_14;
|
|
1209
|
-
}
|
|
1210
|
-
});
|
|
1211
|
-
test('write checkpoint', async () => {
|
|
1212
|
-
const env_15 = { stack: [], error: void 0, hasError: false };
|
|
1213
|
-
try {
|
|
1214
|
-
const f = __addDisposableResource(env_15, await factory(), true);
|
|
1215
|
-
const syncRules = await updateSyncRules(f, {
|
|
1216
|
-
content: BASIC_SYNC_RULES
|
|
1217
|
-
});
|
|
1218
|
-
const bucketStorage = f.getInstance(syncRules);
|
|
1219
|
-
const writer = __addDisposableResource(env_15, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1220
|
-
await writer.markAllSnapshotDone('0/1');
|
|
1221
|
-
// <= the managed write checkpoint LSN below
|
|
1222
|
-
await writer.commit('0/1');
|
|
1223
|
-
const checkpoint = await bucketStorage.createManagedWriteCheckpoint({
|
|
1224
|
-
user_id: 'test',
|
|
1225
|
-
heads: { '1': '1/0' }
|
|
1226
|
-
});
|
|
1227
|
-
const params = {
|
|
1228
|
-
syncContext,
|
|
1229
|
-
bucketStorage,
|
|
1230
|
-
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
1231
|
-
params: {
|
|
1232
|
-
buckets: [],
|
|
1233
|
-
include_checksum: true,
|
|
1234
|
-
raw_data: true
|
|
1235
|
-
},
|
|
1236
|
-
tracker,
|
|
1237
|
-
token: new JwtPayload({ sub: 'test', exp: Date.now() / 1000 + 10 }),
|
|
1238
|
-
isEncodingAsBson: false
|
|
1239
|
-
};
|
|
1240
|
-
const stream1 = sync.streamResponse(params);
|
|
1241
|
-
const lines1 = await consumeCheckpointLines(stream1);
|
|
1242
|
-
// If write checkpoints are not correctly filtered, this may already
|
|
1243
|
-
// contain the write checkpoint.
|
|
1244
|
-
expect(lines1[0]).toMatchObject({
|
|
1245
|
-
checkpoint: expect.objectContaining({
|
|
1246
|
-
last_op_id: '0',
|
|
1247
|
-
write_checkpoint: undefined
|
|
1248
|
-
})
|
|
1249
|
-
});
|
|
1250
|
-
await writer.markAllSnapshotDone('0/1');
|
|
1251
|
-
// must be >= the managed write checkpoint LSN
|
|
1252
|
-
await writer.commit('1/0');
|
|
1253
|
-
// At this point the LSN has advanced, so the write checkpoint should be
|
|
1254
|
-
// included in the next checkpoint message.
|
|
1255
|
-
const stream2 = sync.streamResponse(params);
|
|
1256
|
-
const lines2 = await consumeCheckpointLines(stream2);
|
|
1257
|
-
expect(lines2[0]).toMatchObject({
|
|
1258
|
-
checkpoint: expect.objectContaining({
|
|
1259
|
-
last_op_id: '0',
|
|
1260
|
-
write_checkpoint: `${checkpoint}`
|
|
1261
|
-
})
|
|
1262
|
-
});
|
|
1263
|
-
}
|
|
1264
|
-
catch (e_15) {
|
|
1265
|
-
env_15.error = e_15;
|
|
1266
|
-
env_15.hasError = true;
|
|
1267
|
-
}
|
|
1268
|
-
finally {
|
|
1269
|
-
const result_15 = __disposeResources(env_15);
|
|
1270
|
-
if (result_15)
|
|
1271
|
-
await result_15;
|
|
1272
|
-
}
|
|
1273
|
-
});
|
|
1274
|
-
test('encodes sync rules id in buckets for streams', async () => {
|
|
1275
|
-
const env_16 = { stack: [], error: void 0, hasError: false };
|
|
1276
|
-
try {
|
|
1277
|
-
const f = __addDisposableResource(env_16, await factory(), true);
|
|
1278
|
-
// This test relies making an actual update to sync rules to test the different bucket names.
|
|
1279
|
-
// The actual naming scheme may change, as long as the two buckets have different names.
|
|
1280
|
-
const rules = [
|
|
1281
|
-
`
|
|
1282
|
-
streams:
|
|
1283
|
-
test:
|
|
1284
|
-
auto_subscribe: true
|
|
1285
|
-
query: SELECT * FROM test;
|
|
1286
|
-
|
|
1287
|
-
config:
|
|
1288
|
-
edition: 2
|
|
1289
|
-
`,
|
|
1290
|
-
`
|
|
1291
|
-
streams:
|
|
1292
|
-
test2:
|
|
1293
|
-
auto_subscribe: true
|
|
1294
|
-
query: SELECT * FROM test WHERE 1;
|
|
1295
|
-
|
|
1296
|
-
config:
|
|
1297
|
-
edition: 2
|
|
1298
|
-
`
|
|
1299
|
-
];
|
|
1300
|
-
for (let i = 0; i < 2; i++) {
|
|
1301
|
-
const env_17 = { stack: [], error: void 0, hasError: false };
|
|
1302
|
-
try {
|
|
1303
|
-
const syncRules = await updateSyncRules(f, {
|
|
1304
|
-
content: rules[i]
|
|
1305
|
-
});
|
|
1306
|
-
const bucketStorage = f.getInstance(syncRules);
|
|
1307
|
-
const writer = __addDisposableResource(env_17, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1308
|
-
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1);
|
|
1309
|
-
await writer.markAllSnapshotDone('0/1');
|
|
1310
|
-
await writer.save({
|
|
1311
|
-
sourceTable: testTable,
|
|
1312
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1313
|
-
after: {
|
|
1314
|
-
id: 't1',
|
|
1315
|
-
description: 'Test 1'
|
|
1316
|
-
},
|
|
1317
|
-
afterReplicaId: 't1'
|
|
1318
|
-
});
|
|
1319
|
-
await writer.commit('0/1');
|
|
1320
|
-
const stream = sync.streamResponse({
|
|
1321
|
-
syncContext,
|
|
1322
|
-
bucketStorage: bucketStorage,
|
|
1323
|
-
syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS),
|
|
1324
|
-
params: {
|
|
1325
|
-
buckets: [],
|
|
1326
|
-
include_checksum: true,
|
|
1327
|
-
raw_data: true
|
|
1328
|
-
},
|
|
1329
|
-
tracker,
|
|
1330
|
-
token: new JwtPayload({ sub: '', exp: Date.now() / 1000 + 10 }),
|
|
1331
|
-
isEncodingAsBson: false
|
|
1332
|
-
});
|
|
1333
|
-
const lines = await consumeCheckpointLines(stream);
|
|
1334
|
-
expect(lines).toMatchSnapshot();
|
|
1335
|
-
}
|
|
1336
|
-
catch (e_16) {
|
|
1337
|
-
env_17.error = e_16;
|
|
1338
|
-
env_17.hasError = true;
|
|
1339
|
-
}
|
|
1340
|
-
finally {
|
|
1341
|
-
const result_16 = __disposeResources(env_17);
|
|
1342
|
-
if (result_16)
|
|
1343
|
-
await result_16;
|
|
1344
|
-
}
|
|
1345
|
-
}
|
|
1346
|
-
}
|
|
1347
|
-
catch (e_17) {
|
|
1348
|
-
env_16.error = e_17;
|
|
1349
|
-
env_16.hasError = true;
|
|
1350
|
-
}
|
|
1351
|
-
finally {
|
|
1352
|
-
const result_17 = __disposeResources(env_16);
|
|
1353
|
-
if (result_17)
|
|
1354
|
-
await result_17;
|
|
1355
1165
|
}
|
|
1356
1166
|
});
|
|
1357
1167
|
}
|