@powersync/service-core 1.13.4 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/LICENSE +3 -3
- package/dist/api/api-metrics.js +5 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.js +31 -1
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/KeyStore.d.ts +19 -0
- package/dist/auth/KeyStore.js +16 -4
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.d.ts +3 -0
- package/dist/auth/RemoteJWKSCollector.js +3 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.d.ts +2 -1
- package/dist/auth/StaticSupabaseKeyCollector.js +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/auth/utils.d.ts +19 -0
- package/dist/auth/utils.js +106 -3
- package/dist/auth/utils.js.map +1 -1
- package/dist/entry/commands/compact-action.js +10 -1
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/metrics/open-telemetry/util.d.ts +0 -3
- package/dist/metrics/open-telemetry/util.js +19 -12
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -2
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/compression.d.ts +19 -0
- package/dist/routes/compression.js +70 -0
- package/dist/routes/compression.js.map +1 -0
- package/dist/routes/configure-fastify.d.ts +40 -5
- package/dist/routes/configure-fastify.js +2 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +25 -17
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +1 -27
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +80 -10
- package/dist/routes/endpoints/sync-stream.js +29 -11
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +4 -0
- package/dist/routes/route-register.js +29 -15
- package/dist/routes/route-register.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +1 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +16 -6
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +4 -19
- package/dist/storage/ChecksumCache.js +4 -0
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +2 -2
- package/dist/storage/SourceEntity.d.ts +5 -4
- package/dist/storage/SourceTable.d.ts +22 -20
- package/dist/storage/SourceTable.js +34 -30
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +19 -4
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +41 -11
- package/dist/sync/BucketChecksumState.js +155 -19
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +7 -1
- package/dist/sync/RequestTracker.js +22 -2
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync.d.ts +3 -3
- package/dist/sync/sync.js +23 -42
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +3 -1
- package/dist/sync/util.js +30 -2
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +23 -0
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/lsn.d.ts +4 -0
- package/dist/util/lsn.js +11 -0
- package/dist/util/lsn.js.map +1 -0
- package/dist/util/protocol-types.d.ts +153 -9
- package/dist/util/protocol-types.js +41 -6
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +18 -3
- package/dist/util/utils.js +33 -9
- package/dist/util/utils.js.map +1 -1
- package/package.json +16 -14
- package/src/api/api-metrics.ts +6 -0
- package/src/api/diagnostics.ts +33 -1
- package/src/auth/KeyStore.ts +28 -4
- package/src/auth/RemoteJWKSCollector.ts +5 -2
- package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
- package/src/auth/utils.ts +123 -3
- package/src/entry/commands/compact-action.ts +9 -1
- package/src/metrics/open-telemetry/util.ts +23 -19
- package/src/replication/AbstractReplicator.ts +2 -2
- package/src/routes/compression.ts +75 -0
- package/src/routes/configure-fastify.ts +3 -1
- package/src/routes/endpoints/socket-route.ts +25 -16
- package/src/routes/endpoints/sync-rules.ts +1 -28
- package/src/routes/endpoints/sync-stream.ts +37 -26
- package/src/routes/route-register.ts +41 -15
- package/src/storage/BucketStorage.ts +2 -2
- package/src/storage/BucketStorageBatch.ts +23 -6
- package/src/storage/ChecksumCache.ts +8 -22
- package/src/storage/ReplicationEventPayload.ts +2 -2
- package/src/storage/SourceEntity.ts +5 -5
- package/src/storage/SourceTable.ts +48 -34
- package/src/storage/SyncRulesBucketStorage.ts +26 -7
- package/src/sync/BucketChecksumState.ts +194 -31
- package/src/sync/RequestTracker.ts +27 -2
- package/src/sync/sync.ts +53 -51
- package/src/sync/util.ts +32 -3
- package/src/util/config/compound-config-collector.ts +24 -0
- package/src/util/lsn.ts +8 -0
- package/src/util/protocol-types.ts +138 -10
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +59 -12
- package/test/src/auth.test.ts +323 -1
- package/test/src/checksum_cache.test.ts +6 -8
- package/test/src/routes/mocks.ts +59 -0
- package/test/src/routes/stream.test.ts +84 -0
- package/test/src/sync/BucketChecksumState.test.ts +375 -76
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BucketChecksum,
|
|
3
3
|
BucketChecksumState,
|
|
4
|
+
BucketChecksumStateOptions,
|
|
4
5
|
BucketChecksumStateStorage,
|
|
5
6
|
CHECKPOINT_INVALIDATE_ALL,
|
|
6
7
|
ChecksumMap,
|
|
7
8
|
InternalOpId,
|
|
9
|
+
ReplicationCheckpoint,
|
|
10
|
+
StreamingSyncRequest,
|
|
8
11
|
SyncContext,
|
|
9
12
|
WatchFilterEvent
|
|
10
13
|
} from '@/index.js';
|
|
11
14
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
12
|
-
import {
|
|
13
|
-
|
|
15
|
+
import {
|
|
16
|
+
SqliteJsonRow,
|
|
17
|
+
ParameterLookup,
|
|
18
|
+
SqlSyncRules,
|
|
19
|
+
RequestJwtPayload,
|
|
20
|
+
BucketSource
|
|
21
|
+
} from '@powersync/service-sync-rules';
|
|
22
|
+
import { describe, expect, test, beforeEach } from 'vitest';
|
|
14
23
|
|
|
15
24
|
describe('BucketChecksumState', () => {
|
|
16
25
|
// Single global[] bucket.
|
|
@@ -54,6 +63,9 @@ bucket_definitions:
|
|
|
54
63
|
maxDataFetchConcurrency: 10
|
|
55
64
|
});
|
|
56
65
|
|
|
66
|
+
const syncRequest: StreamingSyncRequest = {};
|
|
67
|
+
const tokenPayload: RequestJwtPayload = { sub: '' };
|
|
68
|
+
|
|
57
69
|
test('global bucket with update', async () => {
|
|
58
70
|
const storage = new MockBucketChecksumStateStorage();
|
|
59
71
|
// Set intial state
|
|
@@ -61,22 +73,27 @@ bucket_definitions:
|
|
|
61
73
|
|
|
62
74
|
const state = new BucketChecksumState({
|
|
63
75
|
syncContext,
|
|
64
|
-
|
|
65
|
-
|
|
76
|
+
syncRequest,
|
|
77
|
+
tokenPayload,
|
|
78
|
+
syncRules: {
|
|
79
|
+
syncRules: SYNC_RULES_GLOBAL,
|
|
80
|
+
version: 1
|
|
81
|
+
},
|
|
66
82
|
bucketStorage: storage
|
|
67
83
|
});
|
|
68
84
|
|
|
69
85
|
const line = (await state.buildNextCheckpointLine({
|
|
70
|
-
base:
|
|
86
|
+
base: storage.makeCheckpoint(1n),
|
|
71
87
|
writeCheckpoint: null,
|
|
72
88
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
73
89
|
}))!;
|
|
74
90
|
line.advance();
|
|
75
91
|
expect(line.checkpointLine).toEqual({
|
|
76
92
|
checkpoint: {
|
|
77
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
|
|
93
|
+
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
78
94
|
last_op_id: '1',
|
|
79
|
-
write_checkpoint: undefined
|
|
95
|
+
write_checkpoint: undefined,
|
|
96
|
+
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
80
97
|
}
|
|
81
98
|
});
|
|
82
99
|
expect(line.bucketsToFetch).toEqual([
|
|
@@ -97,7 +114,7 @@ bucket_definitions:
|
|
|
97
114
|
|
|
98
115
|
// Now we get a new line
|
|
99
116
|
const line2 = (await state.buildNextCheckpointLine({
|
|
100
|
-
base:
|
|
117
|
+
base: storage.makeCheckpoint(2n),
|
|
101
118
|
writeCheckpoint: null,
|
|
102
119
|
update: {
|
|
103
120
|
updatedDataBuckets: new Set(['global[]']),
|
|
@@ -110,7 +127,7 @@ bucket_definitions:
|
|
|
110
127
|
expect(line2.checkpointLine).toEqual({
|
|
111
128
|
checkpoint_diff: {
|
|
112
129
|
removed_buckets: [],
|
|
113
|
-
updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3 }],
|
|
130
|
+
updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
114
131
|
last_op_id: '2',
|
|
115
132
|
write_checkpoint: undefined
|
|
116
133
|
}
|
|
@@ -128,24 +145,28 @@ bucket_definitions:
|
|
|
128
145
|
|
|
129
146
|
const state = new BucketChecksumState({
|
|
130
147
|
syncContext,
|
|
148
|
+
tokenPayload,
|
|
131
149
|
// Client sets the initial state here
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
150
|
+
syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
|
|
151
|
+
syncRules: {
|
|
152
|
+
syncRules: SYNC_RULES_GLOBAL,
|
|
153
|
+
version: 1
|
|
154
|
+
},
|
|
135
155
|
bucketStorage: storage
|
|
136
156
|
});
|
|
137
157
|
|
|
138
158
|
const line = (await state.buildNextCheckpointLine({
|
|
139
|
-
base:
|
|
159
|
+
base: storage.makeCheckpoint(1n),
|
|
140
160
|
writeCheckpoint: null,
|
|
141
161
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
142
162
|
}))!;
|
|
143
163
|
line.advance();
|
|
144
164
|
expect(line.checkpointLine).toEqual({
|
|
145
165
|
checkpoint: {
|
|
146
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
|
|
166
|
+
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
147
167
|
last_op_id: '1',
|
|
148
|
-
write_checkpoint: undefined
|
|
168
|
+
write_checkpoint: undefined,
|
|
169
|
+
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
149
170
|
}
|
|
150
171
|
});
|
|
151
172
|
expect(line.bucketsToFetch).toEqual([
|
|
@@ -166,24 +187,29 @@ bucket_definitions:
|
|
|
166
187
|
|
|
167
188
|
const state = new BucketChecksumState({
|
|
168
189
|
syncContext,
|
|
169
|
-
|
|
170
|
-
|
|
190
|
+
tokenPayload,
|
|
191
|
+
syncRequest,
|
|
192
|
+
syncRules: {
|
|
193
|
+
syncRules: SYNC_RULES_GLOBAL_TWO,
|
|
194
|
+
version: 2
|
|
195
|
+
},
|
|
171
196
|
bucketStorage: storage
|
|
172
197
|
});
|
|
173
198
|
|
|
174
199
|
const line = (await state.buildNextCheckpointLine({
|
|
175
|
-
base:
|
|
200
|
+
base: storage.makeCheckpoint(1n),
|
|
176
201
|
writeCheckpoint: null,
|
|
177
202
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
178
203
|
}))!;
|
|
179
204
|
expect(line.checkpointLine).toEqual({
|
|
180
205
|
checkpoint: {
|
|
181
206
|
buckets: [
|
|
182
|
-
{ bucket: 'global[1]', checksum: 1, count: 1, priority: 3 },
|
|
183
|
-
{ bucket: 'global[2]', checksum: 1, count: 1, priority: 3 }
|
|
207
|
+
{ bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
208
|
+
{ bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
184
209
|
],
|
|
185
210
|
last_op_id: '1',
|
|
186
|
-
write_checkpoint: undefined
|
|
211
|
+
write_checkpoint: undefined,
|
|
212
|
+
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
187
213
|
}
|
|
188
214
|
});
|
|
189
215
|
expect(line.bucketsToFetch).toEqual([
|
|
@@ -202,7 +228,7 @@ bucket_definitions:
|
|
|
202
228
|
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
203
229
|
|
|
204
230
|
const line2 = (await state.buildNextCheckpointLine({
|
|
205
|
-
base:
|
|
231
|
+
base: storage.makeCheckpoint(2n),
|
|
206
232
|
writeCheckpoint: null,
|
|
207
233
|
update: {
|
|
208
234
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
@@ -214,8 +240,8 @@ bucket_definitions:
|
|
|
214
240
|
checkpoint_diff: {
|
|
215
241
|
removed_buckets: [],
|
|
216
242
|
updated_buckets: [
|
|
217
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3 },
|
|
218
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3 }
|
|
243
|
+
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
244
|
+
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
219
245
|
],
|
|
220
246
|
last_op_id: '2',
|
|
221
247
|
write_checkpoint: undefined
|
|
@@ -231,26 +257,30 @@ bucket_definitions:
|
|
|
231
257
|
|
|
232
258
|
const state = new BucketChecksumState({
|
|
233
259
|
syncContext,
|
|
260
|
+
tokenPayload,
|
|
234
261
|
// Client sets the initial state here
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
262
|
+
syncRequest: { buckets: [{ name: 'something_here[]', after: '1' }] },
|
|
263
|
+
syncRules: {
|
|
264
|
+
syncRules: SYNC_RULES_GLOBAL,
|
|
265
|
+
version: 1
|
|
266
|
+
},
|
|
238
267
|
bucketStorage: storage
|
|
239
268
|
});
|
|
240
269
|
|
|
241
270
|
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
242
271
|
|
|
243
272
|
const line = (await state.buildNextCheckpointLine({
|
|
244
|
-
base:
|
|
273
|
+
base: storage.makeCheckpoint(1n),
|
|
245
274
|
writeCheckpoint: null,
|
|
246
275
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
247
276
|
}))!;
|
|
248
277
|
line.advance();
|
|
249
278
|
expect(line.checkpointLine).toEqual({
|
|
250
279
|
checkpoint: {
|
|
251
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
|
|
280
|
+
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
252
281
|
last_op_id: '1',
|
|
253
|
-
write_checkpoint: undefined
|
|
282
|
+
write_checkpoint: undefined,
|
|
283
|
+
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
254
284
|
}
|
|
255
285
|
});
|
|
256
286
|
expect(line.bucketsToFetch).toEqual([
|
|
@@ -272,8 +302,12 @@ bucket_definitions:
|
|
|
272
302
|
|
|
273
303
|
const state = new BucketChecksumState({
|
|
274
304
|
syncContext,
|
|
275
|
-
|
|
276
|
-
|
|
305
|
+
tokenPayload,
|
|
306
|
+
syncRequest,
|
|
307
|
+
syncRules: {
|
|
308
|
+
syncRules: SYNC_RULES_GLOBAL_TWO,
|
|
309
|
+
version: 1
|
|
310
|
+
},
|
|
277
311
|
bucketStorage: storage
|
|
278
312
|
});
|
|
279
313
|
|
|
@@ -281,7 +315,7 @@ bucket_definitions:
|
|
|
281
315
|
// storage.filter = state.checkpointFilter;
|
|
282
316
|
|
|
283
317
|
const line = await state.buildNextCheckpointLine({
|
|
284
|
-
base:
|
|
318
|
+
base: storage.makeCheckpoint(1n),
|
|
285
319
|
writeCheckpoint: null,
|
|
286
320
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
287
321
|
});
|
|
@@ -293,7 +327,7 @@ bucket_definitions:
|
|
|
293
327
|
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
294
328
|
|
|
295
329
|
const line2 = (await state.buildNextCheckpointLine({
|
|
296
|
-
base:
|
|
330
|
+
base: storage.makeCheckpoint(2n),
|
|
297
331
|
writeCheckpoint: null,
|
|
298
332
|
update: {
|
|
299
333
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
@@ -309,7 +343,7 @@ bucket_definitions:
|
|
|
309
343
|
removed_buckets: [],
|
|
310
344
|
updated_buckets: [
|
|
311
345
|
// This does not include global[2], since it was not invalidated.
|
|
312
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3 }
|
|
346
|
+
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
313
347
|
],
|
|
314
348
|
last_op_id: '2',
|
|
315
349
|
write_checkpoint: undefined
|
|
@@ -324,8 +358,12 @@ bucket_definitions:
|
|
|
324
358
|
|
|
325
359
|
const state = new BucketChecksumState({
|
|
326
360
|
syncContext,
|
|
327
|
-
|
|
328
|
-
|
|
361
|
+
tokenPayload,
|
|
362
|
+
syncRequest,
|
|
363
|
+
syncRules: {
|
|
364
|
+
syncRules: SYNC_RULES_GLOBAL_TWO,
|
|
365
|
+
version: 2
|
|
366
|
+
},
|
|
329
367
|
bucketStorage: storage
|
|
330
368
|
});
|
|
331
369
|
|
|
@@ -337,7 +375,7 @@ bucket_definitions:
|
|
|
337
375
|
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
338
376
|
|
|
339
377
|
const line = await state.buildNextCheckpointLine({
|
|
340
|
-
base:
|
|
378
|
+
base: storage.makeCheckpoint(1n),
|
|
341
379
|
writeCheckpoint: null,
|
|
342
380
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
343
381
|
});
|
|
@@ -348,7 +386,7 @@ bucket_definitions:
|
|
|
348
386
|
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
349
387
|
|
|
350
388
|
const line2 = (await state.buildNextCheckpointLine({
|
|
351
|
-
base:
|
|
389
|
+
base: storage.makeCheckpoint(2n),
|
|
352
390
|
writeCheckpoint: null,
|
|
353
391
|
// Invalidate the state - will re-check all buckets
|
|
354
392
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
@@ -357,8 +395,8 @@ bucket_definitions:
|
|
|
357
395
|
checkpoint_diff: {
|
|
358
396
|
removed_buckets: [],
|
|
359
397
|
updated_buckets: [
|
|
360
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3 },
|
|
361
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3 }
|
|
398
|
+
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
399
|
+
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
362
400
|
],
|
|
363
401
|
last_op_id: '2',
|
|
364
402
|
write_checkpoint: undefined
|
|
@@ -378,13 +416,17 @@ bucket_definitions:
|
|
|
378
416
|
|
|
379
417
|
const state = new BucketChecksumState({
|
|
380
418
|
syncContext,
|
|
381
|
-
|
|
382
|
-
|
|
419
|
+
tokenPayload,
|
|
420
|
+
syncRequest,
|
|
421
|
+
syncRules: {
|
|
422
|
+
syncRules: SYNC_RULES_GLOBAL_TWO,
|
|
423
|
+
version: 2
|
|
424
|
+
},
|
|
383
425
|
bucketStorage: storage
|
|
384
426
|
});
|
|
385
427
|
|
|
386
428
|
const line = (await state.buildNextCheckpointLine({
|
|
387
|
-
base:
|
|
429
|
+
base: storage.makeCheckpoint(3n),
|
|
388
430
|
writeCheckpoint: null,
|
|
389
431
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
390
432
|
}))!;
|
|
@@ -392,11 +434,12 @@ bucket_definitions:
|
|
|
392
434
|
expect(line.checkpointLine).toEqual({
|
|
393
435
|
checkpoint: {
|
|
394
436
|
buckets: [
|
|
395
|
-
{ bucket: 'global[1]', checksum: 3, count: 3, priority: 3 },
|
|
396
|
-
{ bucket: 'global[2]', checksum: 3, count: 3, priority: 3 }
|
|
437
|
+
{ bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
438
|
+
{ bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
397
439
|
],
|
|
398
440
|
last_op_id: '3',
|
|
399
|
-
write_checkpoint: undefined
|
|
441
|
+
write_checkpoint: undefined,
|
|
442
|
+
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
400
443
|
}
|
|
401
444
|
});
|
|
402
445
|
expect(line.bucketsToFetch).toEqual([
|
|
@@ -426,7 +469,7 @@ bucket_definitions:
|
|
|
426
469
|
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
|
|
427
470
|
|
|
428
471
|
const line2 = (await state.buildNextCheckpointLine({
|
|
429
|
-
base:
|
|
472
|
+
base: storage.makeCheckpoint(4n),
|
|
430
473
|
writeCheckpoint: null,
|
|
431
474
|
update: {
|
|
432
475
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
@@ -443,7 +486,8 @@ bucket_definitions:
|
|
|
443
486
|
bucket: 'global[1]',
|
|
444
487
|
checksum: 4,
|
|
445
488
|
count: 4,
|
|
446
|
-
priority: 3
|
|
489
|
+
priority: 3,
|
|
490
|
+
subscriptions: [{ default: 0 }]
|
|
447
491
|
}
|
|
448
492
|
],
|
|
449
493
|
last_op_id: '4',
|
|
@@ -479,32 +523,49 @@ bucket_definitions:
|
|
|
479
523
|
|
|
480
524
|
const state = new BucketChecksumState({
|
|
481
525
|
syncContext,
|
|
482
|
-
|
|
483
|
-
|
|
526
|
+
tokenPayload: { sub: 'u1' },
|
|
527
|
+
syncRequest,
|
|
528
|
+
syncRules: {
|
|
529
|
+
syncRules: SYNC_RULES_DYNAMIC,
|
|
530
|
+
version: 1
|
|
531
|
+
},
|
|
484
532
|
bucketStorage: storage
|
|
485
533
|
});
|
|
486
534
|
|
|
487
|
-
storage.getParameterSets = async (
|
|
488
|
-
checkpoint: InternalOpId,
|
|
489
|
-
lookups: ParameterLookup[]
|
|
490
|
-
): Promise<SqliteJsonRow[]> => {
|
|
491
|
-
expect(checkpoint).toEqual(1n);
|
|
492
|
-
expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
|
|
493
|
-
return [{ id: 1 }, { id: 2 }];
|
|
494
|
-
};
|
|
495
|
-
|
|
496
535
|
const line = (await state.buildNextCheckpointLine({
|
|
497
|
-
base:
|
|
536
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
537
|
+
expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
|
|
538
|
+
return [{ id: 1 }, { id: 2 }];
|
|
539
|
+
}),
|
|
498
540
|
writeCheckpoint: null,
|
|
499
541
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
500
542
|
}))!;
|
|
501
543
|
expect(line.checkpointLine).toEqual({
|
|
502
544
|
checkpoint: {
|
|
503
545
|
buckets: [
|
|
504
|
-
{
|
|
505
|
-
|
|
546
|
+
{
|
|
547
|
+
bucket: 'by_project[1]',
|
|
548
|
+
checksum: 1,
|
|
549
|
+
count: 1,
|
|
550
|
+
priority: 3,
|
|
551
|
+
subscriptions: [{ default: 0 }]
|
|
552
|
+
},
|
|
553
|
+
{
|
|
554
|
+
bucket: 'by_project[2]',
|
|
555
|
+
checksum: 1,
|
|
556
|
+
count: 1,
|
|
557
|
+
priority: 3,
|
|
558
|
+
subscriptions: [{ default: 0 }]
|
|
559
|
+
}
|
|
506
560
|
],
|
|
507
561
|
last_op_id: '1',
|
|
562
|
+
streams: [
|
|
563
|
+
{
|
|
564
|
+
is_default: true,
|
|
565
|
+
name: 'by_project',
|
|
566
|
+
errors: []
|
|
567
|
+
}
|
|
568
|
+
],
|
|
508
569
|
write_checkpoint: undefined
|
|
509
570
|
}
|
|
510
571
|
});
|
|
@@ -531,18 +592,12 @@ bucket_definitions:
|
|
|
531
592
|
line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
|
|
532
593
|
line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
|
|
533
594
|
|
|
534
|
-
storage.getParameterSets = async (
|
|
535
|
-
checkpoint: InternalOpId,
|
|
536
|
-
lookups: ParameterLookup[]
|
|
537
|
-
): Promise<SqliteJsonRow[]> => {
|
|
538
|
-
expect(checkpoint).toEqual(2n);
|
|
539
|
-
expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
|
|
540
|
-
return [{ id: 1 }, { id: 2 }, { id: 3 }];
|
|
541
|
-
};
|
|
542
|
-
|
|
543
595
|
// Now we get a new line
|
|
544
596
|
const line2 = (await state.buildNextCheckpointLine({
|
|
545
|
-
base:
|
|
597
|
+
base: storage.makeCheckpoint(2n, (lookups) => {
|
|
598
|
+
expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
|
|
599
|
+
return [{ id: 1 }, { id: 2 }, { id: 3 }];
|
|
600
|
+
}),
|
|
546
601
|
writeCheckpoint: null,
|
|
547
602
|
update: {
|
|
548
603
|
invalidateDataBuckets: false,
|
|
@@ -555,13 +610,245 @@ bucket_definitions:
|
|
|
555
610
|
expect(line2.checkpointLine).toEqual({
|
|
556
611
|
checkpoint_diff: {
|
|
557
612
|
removed_buckets: [],
|
|
558
|
-
updated_buckets: [
|
|
613
|
+
updated_buckets: [
|
|
614
|
+
{
|
|
615
|
+
bucket: 'by_project[3]',
|
|
616
|
+
checksum: 1,
|
|
617
|
+
count: 1,
|
|
618
|
+
priority: 3,
|
|
619
|
+
subscriptions: [{ default: 0 }]
|
|
620
|
+
}
|
|
621
|
+
],
|
|
559
622
|
last_op_id: '2',
|
|
560
623
|
write_checkpoint: undefined
|
|
561
624
|
}
|
|
562
625
|
});
|
|
563
626
|
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
|
|
564
627
|
});
|
|
628
|
+
|
|
629
|
+
describe('streams', () => {
|
|
630
|
+
let source: { -readonly [P in keyof BucketSource]: BucketSource[P] };
|
|
631
|
+
let storage: MockBucketChecksumStateStorage;
|
|
632
|
+
|
|
633
|
+
function checksumState(source: string | boolean, options?: Partial<BucketChecksumStateOptions>) {
|
|
634
|
+
if (typeof source == 'boolean') {
|
|
635
|
+
source = `
|
|
636
|
+
streams:
|
|
637
|
+
stream:
|
|
638
|
+
auto_subscribe: ${source}
|
|
639
|
+
query: SELECT * FROM assets WHERE id IN ifnull(subscription.parameter('ids'), '["default"]');
|
|
640
|
+
|
|
641
|
+
config:
|
|
642
|
+
edition: 2
|
|
643
|
+
`;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
const rules = SqlSyncRules.fromYaml(source, {
|
|
647
|
+
defaultSchema: 'public'
|
|
648
|
+
});
|
|
649
|
+
|
|
650
|
+
return new BucketChecksumState({
|
|
651
|
+
syncContext,
|
|
652
|
+
syncRequest,
|
|
653
|
+
tokenPayload,
|
|
654
|
+
syncRules: { syncRules: rules, version: 1 },
|
|
655
|
+
bucketStorage: storage,
|
|
656
|
+
...options
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
beforeEach(() => {
|
|
661
|
+
storage = new MockBucketChecksumStateStorage();
|
|
662
|
+
storage.updateTestChecksum({ bucket: '1#stream|0["default"]', checksum: 1, count: 1 });
|
|
663
|
+
storage.updateTestChecksum({ bucket: '1#stream|0["a"]', checksum: 1, count: 1 });
|
|
664
|
+
storage.updateTestChecksum({ bucket: '1#stream|0["b"]', checksum: 1, count: 1 });
|
|
665
|
+
});
|
|
666
|
+
|
|
667
|
+
test('includes defaults', async () => {
|
|
668
|
+
const state = checksumState(true);
|
|
669
|
+
const line = await state.buildNextCheckpointLine({
|
|
670
|
+
base: storage.makeCheckpoint(1n),
|
|
671
|
+
writeCheckpoint: null,
|
|
672
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
673
|
+
})!;
|
|
674
|
+
line?.advance();
|
|
675
|
+
expect(line?.checkpointLine).toEqual({
|
|
676
|
+
checkpoint: {
|
|
677
|
+
buckets: [
|
|
678
|
+
{ bucket: '1#stream|0["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
679
|
+
],
|
|
680
|
+
last_op_id: '1',
|
|
681
|
+
write_checkpoint: undefined,
|
|
682
|
+
streams: [{ name: 'stream', is_default: true, errors: [] }]
|
|
683
|
+
}
|
|
684
|
+
});
|
|
685
|
+
});
|
|
686
|
+
|
|
687
|
+
test('can exclude defaults', async () => {
|
|
688
|
+
const state = checksumState(true, { syncRequest: { streams: { include_defaults: false, subscriptions: [] } } });
|
|
689
|
+
|
|
690
|
+
const line = await state.buildNextCheckpointLine({
|
|
691
|
+
base: storage.makeCheckpoint(1n),
|
|
692
|
+
writeCheckpoint: null,
|
|
693
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
694
|
+
})!;
|
|
695
|
+
line?.advance();
|
|
696
|
+
expect(line?.checkpointLine).toEqual({
|
|
697
|
+
checkpoint: {
|
|
698
|
+
buckets: [],
|
|
699
|
+
last_op_id: '1',
|
|
700
|
+
write_checkpoint: undefined,
|
|
701
|
+
streams: []
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
test('custom subscriptions', async () => {
|
|
707
|
+
const state = checksumState(true, {
|
|
708
|
+
syncRequest: {
|
|
709
|
+
streams: {
|
|
710
|
+
subscriptions: [
|
|
711
|
+
{ stream: 'stream', parameters: { ids: '["a"]' }, override_priority: null },
|
|
712
|
+
{ stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 }
|
|
713
|
+
]
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
const line = await state.buildNextCheckpointLine({
|
|
719
|
+
base: storage.makeCheckpoint(1n),
|
|
720
|
+
writeCheckpoint: null,
|
|
721
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
722
|
+
})!;
|
|
723
|
+
line?.advance();
|
|
724
|
+
expect(line?.checkpointLine).toEqual({
|
|
725
|
+
checkpoint: {
|
|
726
|
+
buckets: [
|
|
727
|
+
{ bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
|
|
728
|
+
{ bucket: '1#stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 1 }] },
|
|
729
|
+
{ bucket: '1#stream|0["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
730
|
+
],
|
|
731
|
+
last_op_id: '1',
|
|
732
|
+
write_checkpoint: undefined,
|
|
733
|
+
streams: [{ name: 'stream', is_default: true, errors: [] }]
|
|
734
|
+
}
|
|
735
|
+
});
|
|
736
|
+
});
|
|
737
|
+
|
|
738
|
+
test('overlap between custom subscriptions', async () => {
|
|
739
|
+
const state = checksumState(false, {
|
|
740
|
+
syncRequest: {
|
|
741
|
+
streams: {
|
|
742
|
+
subscriptions: [
|
|
743
|
+
{ stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: null },
|
|
744
|
+
{ stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 }
|
|
745
|
+
]
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
});
|
|
749
|
+
|
|
750
|
+
const line = await state.buildNextCheckpointLine({
|
|
751
|
+
base: storage.makeCheckpoint(1n),
|
|
752
|
+
writeCheckpoint: null,
|
|
753
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
754
|
+
})!;
|
|
755
|
+
line?.advance();
|
|
756
|
+
expect(line?.checkpointLine).toEqual({
|
|
757
|
+
checkpoint: {
|
|
758
|
+
buckets: [
|
|
759
|
+
{ bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
|
|
760
|
+
{ bucket: '1#stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }, { sub: 1 }] }
|
|
761
|
+
],
|
|
762
|
+
last_op_id: '1',
|
|
763
|
+
write_checkpoint: undefined,
|
|
764
|
+
streams: [{ name: 'stream', is_default: false, errors: [] }]
|
|
765
|
+
}
|
|
766
|
+
});
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
test('overlap between default and custom subscription', async () => {
|
|
770
|
+
const state = checksumState(true, {
|
|
771
|
+
syncRequest: {
|
|
772
|
+
streams: {
|
|
773
|
+
subscriptions: [{ stream: 'stream', parameters: { ids: '["a", "default"]' }, override_priority: 1 }]
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
});
|
|
777
|
+
|
|
778
|
+
const line = await state.buildNextCheckpointLine({
|
|
779
|
+
base: storage.makeCheckpoint(1n),
|
|
780
|
+
writeCheckpoint: null,
|
|
781
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
782
|
+
})!;
|
|
783
|
+
line?.advance();
|
|
784
|
+
expect(line?.checkpointLine).toEqual({
|
|
785
|
+
checkpoint: {
|
|
786
|
+
buckets: [
|
|
787
|
+
{ bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
|
|
788
|
+
{
|
|
789
|
+
bucket: '1#stream|0["default"]',
|
|
790
|
+
checksum: 1,
|
|
791
|
+
count: 1,
|
|
792
|
+
priority: 1,
|
|
793
|
+
subscriptions: [{ sub: 0 }, { default: 0 }]
|
|
794
|
+
}
|
|
795
|
+
],
|
|
796
|
+
last_op_id: '1',
|
|
797
|
+
write_checkpoint: undefined,
|
|
798
|
+
streams: [{ name: 'stream', is_default: true, errors: [] }]
|
|
799
|
+
}
|
|
800
|
+
});
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
test('reports errors', async () => {
|
|
804
|
+
const state = checksumState(true, {
|
|
805
|
+
syncRequest: {
|
|
806
|
+
streams: {
|
|
807
|
+
subscriptions: [
|
|
808
|
+
{ stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: 1 },
|
|
809
|
+
{ stream: 'stream', parameters: { ids: 'invalid json' }, override_priority: null }
|
|
810
|
+
]
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
});
|
|
814
|
+
|
|
815
|
+
const line = await state.buildNextCheckpointLine({
|
|
816
|
+
base: storage.makeCheckpoint(1n),
|
|
817
|
+
writeCheckpoint: null,
|
|
818
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
819
|
+
})!;
|
|
820
|
+
line?.advance();
|
|
821
|
+
expect(line?.checkpointLine).toEqual({
|
|
822
|
+
checkpoint: {
|
|
823
|
+
buckets: [
|
|
824
|
+
{ bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
|
|
825
|
+
{ bucket: '1#stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
|
|
826
|
+
{
|
|
827
|
+
bucket: '1#stream|0["default"]',
|
|
828
|
+
checksum: 1,
|
|
829
|
+
count: 1,
|
|
830
|
+
priority: 3,
|
|
831
|
+
subscriptions: [{ default: 0 }]
|
|
832
|
+
}
|
|
833
|
+
],
|
|
834
|
+
last_op_id: '1',
|
|
835
|
+
write_checkpoint: undefined,
|
|
836
|
+
streams: [
|
|
837
|
+
{
|
|
838
|
+
name: 'stream',
|
|
839
|
+
is_default: true,
|
|
840
|
+
errors: [
|
|
841
|
+
{
|
|
842
|
+
message: 'Error evaluating bucket ids: Unexpected token \'i\', "invalid json" is not valid JSON',
|
|
843
|
+
subscription: 1
|
|
844
|
+
}
|
|
845
|
+
]
|
|
846
|
+
}
|
|
847
|
+
]
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
});
|
|
851
|
+
});
|
|
565
852
|
});
|
|
566
853
|
|
|
567
854
|
class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
|
|
@@ -595,7 +882,19 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
|
|
|
595
882
|
);
|
|
596
883
|
}
|
|
597
884
|
|
|
598
|
-
|
|
599
|
-
|
|
885
|
+
makeCheckpoint(
|
|
886
|
+
opId: InternalOpId,
|
|
887
|
+
parameters?: (lookups: ParameterLookup[]) => SqliteJsonRow[]
|
|
888
|
+
): ReplicationCheckpoint {
|
|
889
|
+
return {
|
|
890
|
+
checkpoint: opId,
|
|
891
|
+
lsn: String(opId),
|
|
892
|
+
getParameterSets: async (lookups: ParameterLookup[]) => {
|
|
893
|
+
if (parameters == null) {
|
|
894
|
+
throw new Error(`getParametersSets not defined for checkpoint ${opId}`);
|
|
895
|
+
}
|
|
896
|
+
return parameters(lookups);
|
|
897
|
+
}
|
|
898
|
+
};
|
|
600
899
|
}
|
|
601
900
|
}
|