@powersync/service-core 1.13.4 → 1.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +61 -0
  2. package/LICENSE +3 -3
  3. package/dist/api/api-metrics.js +5 -0
  4. package/dist/api/api-metrics.js.map +1 -1
  5. package/dist/api/diagnostics.js +31 -1
  6. package/dist/api/diagnostics.js.map +1 -1
  7. package/dist/auth/KeyStore.d.ts +19 -0
  8. package/dist/auth/KeyStore.js +16 -4
  9. package/dist/auth/KeyStore.js.map +1 -1
  10. package/dist/auth/RemoteJWKSCollector.d.ts +3 -0
  11. package/dist/auth/RemoteJWKSCollector.js +3 -1
  12. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  13. package/dist/auth/StaticSupabaseKeyCollector.d.ts +2 -1
  14. package/dist/auth/StaticSupabaseKeyCollector.js +1 -1
  15. package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
  16. package/dist/auth/utils.d.ts +19 -0
  17. package/dist/auth/utils.js +106 -3
  18. package/dist/auth/utils.js.map +1 -1
  19. package/dist/entry/commands/compact-action.js +10 -1
  20. package/dist/entry/commands/compact-action.js.map +1 -1
  21. package/dist/metrics/open-telemetry/util.d.ts +0 -3
  22. package/dist/metrics/open-telemetry/util.js +19 -12
  23. package/dist/metrics/open-telemetry/util.js.map +1 -1
  24. package/dist/replication/AbstractReplicator.js +2 -2
  25. package/dist/replication/AbstractReplicator.js.map +1 -1
  26. package/dist/routes/compression.d.ts +19 -0
  27. package/dist/routes/compression.js +70 -0
  28. package/dist/routes/compression.js.map +1 -0
  29. package/dist/routes/configure-fastify.d.ts +40 -5
  30. package/dist/routes/configure-fastify.js +2 -1
  31. package/dist/routes/configure-fastify.js.map +1 -1
  32. package/dist/routes/endpoints/socket-route.js +25 -17
  33. package/dist/routes/endpoints/socket-route.js.map +1 -1
  34. package/dist/routes/endpoints/sync-rules.js +1 -27
  35. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  36. package/dist/routes/endpoints/sync-stream.d.ts +80 -10
  37. package/dist/routes/endpoints/sync-stream.js +29 -11
  38. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  39. package/dist/routes/route-register.d.ts +4 -0
  40. package/dist/routes/route-register.js +29 -15
  41. package/dist/routes/route-register.js.map +1 -1
  42. package/dist/storage/BucketStorage.d.ts +1 -1
  43. package/dist/storage/BucketStorage.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +16 -6
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/ChecksumCache.d.ts +4 -19
  47. package/dist/storage/ChecksumCache.js +4 -0
  48. package/dist/storage/ChecksumCache.js.map +1 -1
  49. package/dist/storage/ReplicationEventPayload.d.ts +2 -2
  50. package/dist/storage/SourceEntity.d.ts +5 -4
  51. package/dist/storage/SourceTable.d.ts +22 -20
  52. package/dist/storage/SourceTable.js +34 -30
  53. package/dist/storage/SourceTable.js.map +1 -1
  54. package/dist/storage/SyncRulesBucketStorage.d.ts +19 -4
  55. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  56. package/dist/sync/BucketChecksumState.d.ts +41 -11
  57. package/dist/sync/BucketChecksumState.js +155 -19
  58. package/dist/sync/BucketChecksumState.js.map +1 -1
  59. package/dist/sync/RequestTracker.d.ts +7 -1
  60. package/dist/sync/RequestTracker.js +22 -2
  61. package/dist/sync/RequestTracker.js.map +1 -1
  62. package/dist/sync/sync.d.ts +3 -3
  63. package/dist/sync/sync.js +23 -42
  64. package/dist/sync/sync.js.map +1 -1
  65. package/dist/sync/util.d.ts +3 -1
  66. package/dist/sync/util.js +30 -2
  67. package/dist/sync/util.js.map +1 -1
  68. package/dist/util/config/compound-config-collector.js +23 -0
  69. package/dist/util/config/compound-config-collector.js.map +1 -1
  70. package/dist/util/lsn.d.ts +4 -0
  71. package/dist/util/lsn.js +11 -0
  72. package/dist/util/lsn.js.map +1 -0
  73. package/dist/util/protocol-types.d.ts +153 -9
  74. package/dist/util/protocol-types.js +41 -6
  75. package/dist/util/protocol-types.js.map +1 -1
  76. package/dist/util/util-index.d.ts +1 -0
  77. package/dist/util/util-index.js +1 -0
  78. package/dist/util/util-index.js.map +1 -1
  79. package/dist/util/utils.d.ts +18 -3
  80. package/dist/util/utils.js +33 -9
  81. package/dist/util/utils.js.map +1 -1
  82. package/package.json +16 -14
  83. package/src/api/api-metrics.ts +6 -0
  84. package/src/api/diagnostics.ts +33 -1
  85. package/src/auth/KeyStore.ts +28 -4
  86. package/src/auth/RemoteJWKSCollector.ts +5 -2
  87. package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
  88. package/src/auth/utils.ts +123 -3
  89. package/src/entry/commands/compact-action.ts +9 -1
  90. package/src/metrics/open-telemetry/util.ts +23 -19
  91. package/src/replication/AbstractReplicator.ts +2 -2
  92. package/src/routes/compression.ts +75 -0
  93. package/src/routes/configure-fastify.ts +3 -1
  94. package/src/routes/endpoints/socket-route.ts +25 -16
  95. package/src/routes/endpoints/sync-rules.ts +1 -28
  96. package/src/routes/endpoints/sync-stream.ts +37 -26
  97. package/src/routes/route-register.ts +41 -15
  98. package/src/storage/BucketStorage.ts +2 -2
  99. package/src/storage/BucketStorageBatch.ts +23 -6
  100. package/src/storage/ChecksumCache.ts +8 -22
  101. package/src/storage/ReplicationEventPayload.ts +2 -2
  102. package/src/storage/SourceEntity.ts +5 -5
  103. package/src/storage/SourceTable.ts +48 -34
  104. package/src/storage/SyncRulesBucketStorage.ts +26 -7
  105. package/src/sync/BucketChecksumState.ts +194 -31
  106. package/src/sync/RequestTracker.ts +27 -2
  107. package/src/sync/sync.ts +53 -51
  108. package/src/sync/util.ts +32 -3
  109. package/src/util/config/compound-config-collector.ts +24 -0
  110. package/src/util/lsn.ts +8 -0
  111. package/src/util/protocol-types.ts +138 -10
  112. package/src/util/util-index.ts +1 -0
  113. package/src/util/utils.ts +59 -12
  114. package/test/src/auth.test.ts +323 -1
  115. package/test/src/checksum_cache.test.ts +6 -8
  116. package/test/src/routes/mocks.ts +59 -0
  117. package/test/src/routes/stream.test.ts +84 -0
  118. package/test/src/sync/BucketChecksumState.test.ts +375 -76
  119. package/tsconfig.tsbuildinfo +1 -1
@@ -1,16 +1,25 @@
1
1
  import {
2
2
  BucketChecksum,
3
3
  BucketChecksumState,
4
+ BucketChecksumStateOptions,
4
5
  BucketChecksumStateStorage,
5
6
  CHECKPOINT_INVALIDATE_ALL,
6
7
  ChecksumMap,
7
8
  InternalOpId,
9
+ ReplicationCheckpoint,
10
+ StreamingSyncRequest,
8
11
  SyncContext,
9
12
  WatchFilterEvent
10
13
  } from '@/index.js';
11
14
  import { JSONBig } from '@powersync/service-jsonbig';
12
- import { RequestParameters, SqliteJsonRow, ParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
13
- import { describe, expect, test } from 'vitest';
15
+ import {
16
+ SqliteJsonRow,
17
+ ParameterLookup,
18
+ SqlSyncRules,
19
+ RequestJwtPayload,
20
+ BucketSource
21
+ } from '@powersync/service-sync-rules';
22
+ import { describe, expect, test, beforeEach } from 'vitest';
14
23
 
15
24
  describe('BucketChecksumState', () => {
16
25
  // Single global[] bucket.
@@ -54,6 +63,9 @@ bucket_definitions:
54
63
  maxDataFetchConcurrency: 10
55
64
  });
56
65
 
66
+ const syncRequest: StreamingSyncRequest = {};
67
+ const tokenPayload: RequestJwtPayload = { sub: '' };
68
+
57
69
  test('global bucket with update', async () => {
58
70
  const storage = new MockBucketChecksumStateStorage();
59
71
  // Set intial state
@@ -61,22 +73,27 @@ bucket_definitions:
61
73
 
62
74
  const state = new BucketChecksumState({
63
75
  syncContext,
64
- syncParams: new RequestParameters({ sub: '' }, {}),
65
- syncRules: SYNC_RULES_GLOBAL,
76
+ syncRequest,
77
+ tokenPayload,
78
+ syncRules: {
79
+ syncRules: SYNC_RULES_GLOBAL,
80
+ version: 1
81
+ },
66
82
  bucketStorage: storage
67
83
  });
68
84
 
69
85
  const line = (await state.buildNextCheckpointLine({
70
- base: { checkpoint: 1n, lsn: '1' },
86
+ base: storage.makeCheckpoint(1n),
71
87
  writeCheckpoint: null,
72
88
  update: CHECKPOINT_INVALIDATE_ALL
73
89
  }))!;
74
90
  line.advance();
75
91
  expect(line.checkpointLine).toEqual({
76
92
  checkpoint: {
77
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
93
+ buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
78
94
  last_op_id: '1',
79
- write_checkpoint: undefined
95
+ write_checkpoint: undefined,
96
+ streams: [{ name: 'global', is_default: true, errors: [] }]
80
97
  }
81
98
  });
82
99
  expect(line.bucketsToFetch).toEqual([
@@ -97,7 +114,7 @@ bucket_definitions:
97
114
 
98
115
  // Now we get a new line
99
116
  const line2 = (await state.buildNextCheckpointLine({
100
- base: { checkpoint: 2n, lsn: '2' },
117
+ base: storage.makeCheckpoint(2n),
101
118
  writeCheckpoint: null,
102
119
  update: {
103
120
  updatedDataBuckets: new Set(['global[]']),
@@ -110,7 +127,7 @@ bucket_definitions:
110
127
  expect(line2.checkpointLine).toEqual({
111
128
  checkpoint_diff: {
112
129
  removed_buckets: [],
113
- updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3 }],
130
+ updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }],
114
131
  last_op_id: '2',
115
132
  write_checkpoint: undefined
116
133
  }
@@ -128,24 +145,28 @@ bucket_definitions:
128
145
 
129
146
  const state = new BucketChecksumState({
130
147
  syncContext,
148
+ tokenPayload,
131
149
  // Client sets the initial state here
132
- initialBucketPositions: [{ name: 'global[]', after: 1n }],
133
- syncParams: new RequestParameters({ sub: '' }, {}),
134
- syncRules: SYNC_RULES_GLOBAL,
150
+ syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
151
+ syncRules: {
152
+ syncRules: SYNC_RULES_GLOBAL,
153
+ version: 1
154
+ },
135
155
  bucketStorage: storage
136
156
  });
137
157
 
138
158
  const line = (await state.buildNextCheckpointLine({
139
- base: { checkpoint: 1n, lsn: '1' },
159
+ base: storage.makeCheckpoint(1n),
140
160
  writeCheckpoint: null,
141
161
  update: CHECKPOINT_INVALIDATE_ALL
142
162
  }))!;
143
163
  line.advance();
144
164
  expect(line.checkpointLine).toEqual({
145
165
  checkpoint: {
146
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
166
+ buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
147
167
  last_op_id: '1',
148
- write_checkpoint: undefined
168
+ write_checkpoint: undefined,
169
+ streams: [{ name: 'global', is_default: true, errors: [] }]
149
170
  }
150
171
  });
151
172
  expect(line.bucketsToFetch).toEqual([
@@ -166,24 +187,29 @@ bucket_definitions:
166
187
 
167
188
  const state = new BucketChecksumState({
168
189
  syncContext,
169
- syncParams: new RequestParameters({ sub: '' }, {}),
170
- syncRules: SYNC_RULES_GLOBAL_TWO,
190
+ tokenPayload,
191
+ syncRequest,
192
+ syncRules: {
193
+ syncRules: SYNC_RULES_GLOBAL_TWO,
194
+ version: 2
195
+ },
171
196
  bucketStorage: storage
172
197
  });
173
198
 
174
199
  const line = (await state.buildNextCheckpointLine({
175
- base: { checkpoint: 1n, lsn: '1' },
200
+ base: storage.makeCheckpoint(1n),
176
201
  writeCheckpoint: null,
177
202
  update: CHECKPOINT_INVALIDATE_ALL
178
203
  }))!;
179
204
  expect(line.checkpointLine).toEqual({
180
205
  checkpoint: {
181
206
  buckets: [
182
- { bucket: 'global[1]', checksum: 1, count: 1, priority: 3 },
183
- { bucket: 'global[2]', checksum: 1, count: 1, priority: 3 }
207
+ { bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
208
+ { bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
184
209
  ],
185
210
  last_op_id: '1',
186
- write_checkpoint: undefined
211
+ write_checkpoint: undefined,
212
+ streams: [{ name: 'global', is_default: true, errors: [] }]
187
213
  }
188
214
  });
189
215
  expect(line.bucketsToFetch).toEqual([
@@ -202,7 +228,7 @@ bucket_definitions:
202
228
  storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
203
229
 
204
230
  const line2 = (await state.buildNextCheckpointLine({
205
- base: { checkpoint: 2n, lsn: '2' },
231
+ base: storage.makeCheckpoint(2n),
206
232
  writeCheckpoint: null,
207
233
  update: {
208
234
  ...CHECKPOINT_INVALIDATE_ALL,
@@ -214,8 +240,8 @@ bucket_definitions:
214
240
  checkpoint_diff: {
215
241
  removed_buckets: [],
216
242
  updated_buckets: [
217
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 },
218
- { bucket: 'global[2]', checksum: 2, count: 2, priority: 3 }
243
+ { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
244
+ { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
219
245
  ],
220
246
  last_op_id: '2',
221
247
  write_checkpoint: undefined
@@ -231,26 +257,30 @@ bucket_definitions:
231
257
 
232
258
  const state = new BucketChecksumState({
233
259
  syncContext,
260
+ tokenPayload,
234
261
  // Client sets the initial state here
235
- initialBucketPositions: [{ name: 'something_here[]', after: 1n }],
236
- syncParams: new RequestParameters({ sub: '' }, {}),
237
- syncRules: SYNC_RULES_GLOBAL,
262
+ syncRequest: { buckets: [{ name: 'something_here[]', after: '1' }] },
263
+ syncRules: {
264
+ syncRules: SYNC_RULES_GLOBAL,
265
+ version: 1
266
+ },
238
267
  bucketStorage: storage
239
268
  });
240
269
 
241
270
  storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
242
271
 
243
272
  const line = (await state.buildNextCheckpointLine({
244
- base: { checkpoint: 1n, lsn: '1' },
273
+ base: storage.makeCheckpoint(1n),
245
274
  writeCheckpoint: null,
246
275
  update: CHECKPOINT_INVALIDATE_ALL
247
276
  }))!;
248
277
  line.advance();
249
278
  expect(line.checkpointLine).toEqual({
250
279
  checkpoint: {
251
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
280
+ buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
252
281
  last_op_id: '1',
253
- write_checkpoint: undefined
282
+ write_checkpoint: undefined,
283
+ streams: [{ name: 'global', is_default: true, errors: [] }]
254
284
  }
255
285
  });
256
286
  expect(line.bucketsToFetch).toEqual([
@@ -272,8 +302,12 @@ bucket_definitions:
272
302
 
273
303
  const state = new BucketChecksumState({
274
304
  syncContext,
275
- syncParams: new RequestParameters({ sub: '' }, {}),
276
- syncRules: SYNC_RULES_GLOBAL_TWO,
305
+ tokenPayload,
306
+ syncRequest,
307
+ syncRules: {
308
+ syncRules: SYNC_RULES_GLOBAL_TWO,
309
+ version: 1
310
+ },
277
311
  bucketStorage: storage
278
312
  });
279
313
 
@@ -281,7 +315,7 @@ bucket_definitions:
281
315
  // storage.filter = state.checkpointFilter;
282
316
 
283
317
  const line = await state.buildNextCheckpointLine({
284
- base: { checkpoint: 1n, lsn: '1' },
318
+ base: storage.makeCheckpoint(1n),
285
319
  writeCheckpoint: null,
286
320
  update: CHECKPOINT_INVALIDATE_ALL
287
321
  });
@@ -293,7 +327,7 @@ bucket_definitions:
293
327
  storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
294
328
 
295
329
  const line2 = (await state.buildNextCheckpointLine({
296
- base: { checkpoint: 2n, lsn: '2' },
330
+ base: storage.makeCheckpoint(2n),
297
331
  writeCheckpoint: null,
298
332
  update: {
299
333
  ...CHECKPOINT_INVALIDATE_ALL,
@@ -309,7 +343,7 @@ bucket_definitions:
309
343
  removed_buckets: [],
310
344
  updated_buckets: [
311
345
  // This does not include global[2], since it was not invalidated.
312
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 }
346
+ { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
313
347
  ],
314
348
  last_op_id: '2',
315
349
  write_checkpoint: undefined
@@ -324,8 +358,12 @@ bucket_definitions:
324
358
 
325
359
  const state = new BucketChecksumState({
326
360
  syncContext,
327
- syncParams: new RequestParameters({ sub: '' }, {}),
328
- syncRules: SYNC_RULES_GLOBAL_TWO,
361
+ tokenPayload,
362
+ syncRequest,
363
+ syncRules: {
364
+ syncRules: SYNC_RULES_GLOBAL_TWO,
365
+ version: 2
366
+ },
329
367
  bucketStorage: storage
330
368
  });
331
369
 
@@ -337,7 +375,7 @@ bucket_definitions:
337
375
  storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
338
376
 
339
377
  const line = await state.buildNextCheckpointLine({
340
- base: { checkpoint: 1n, lsn: '1' },
378
+ base: storage.makeCheckpoint(1n),
341
379
  writeCheckpoint: null,
342
380
  update: CHECKPOINT_INVALIDATE_ALL
343
381
  });
@@ -348,7 +386,7 @@ bucket_definitions:
348
386
  storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
349
387
 
350
388
  const line2 = (await state.buildNextCheckpointLine({
351
- base: { checkpoint: 2n, lsn: '2' },
389
+ base: storage.makeCheckpoint(2n),
352
390
  writeCheckpoint: null,
353
391
  // Invalidate the state - will re-check all buckets
354
392
  update: CHECKPOINT_INVALIDATE_ALL
@@ -357,8 +395,8 @@ bucket_definitions:
357
395
  checkpoint_diff: {
358
396
  removed_buckets: [],
359
397
  updated_buckets: [
360
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 },
361
- { bucket: 'global[2]', checksum: 2, count: 2, priority: 3 }
398
+ { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
399
+ { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
362
400
  ],
363
401
  last_op_id: '2',
364
402
  write_checkpoint: undefined
@@ -378,13 +416,17 @@ bucket_definitions:
378
416
 
379
417
  const state = new BucketChecksumState({
380
418
  syncContext,
381
- syncParams: new RequestParameters({ sub: '' }, {}),
382
- syncRules: SYNC_RULES_GLOBAL_TWO,
419
+ tokenPayload,
420
+ syncRequest,
421
+ syncRules: {
422
+ syncRules: SYNC_RULES_GLOBAL_TWO,
423
+ version: 2
424
+ },
383
425
  bucketStorage: storage
384
426
  });
385
427
 
386
428
  const line = (await state.buildNextCheckpointLine({
387
- base: { checkpoint: 3n, lsn: '3' },
429
+ base: storage.makeCheckpoint(3n),
388
430
  writeCheckpoint: null,
389
431
  update: CHECKPOINT_INVALIDATE_ALL
390
432
  }))!;
@@ -392,11 +434,12 @@ bucket_definitions:
392
434
  expect(line.checkpointLine).toEqual({
393
435
  checkpoint: {
394
436
  buckets: [
395
- { bucket: 'global[1]', checksum: 3, count: 3, priority: 3 },
396
- { bucket: 'global[2]', checksum: 3, count: 3, priority: 3 }
437
+ { bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
438
+ { bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
397
439
  ],
398
440
  last_op_id: '3',
399
- write_checkpoint: undefined
441
+ write_checkpoint: undefined,
442
+ streams: [{ name: 'global', is_default: true, errors: [] }]
400
443
  }
401
444
  });
402
445
  expect(line.bucketsToFetch).toEqual([
@@ -426,7 +469,7 @@ bucket_definitions:
426
469
  storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
427
470
 
428
471
  const line2 = (await state.buildNextCheckpointLine({
429
- base: { checkpoint: 4n, lsn: '4' },
472
+ base: storage.makeCheckpoint(4n),
430
473
  writeCheckpoint: null,
431
474
  update: {
432
475
  ...CHECKPOINT_INVALIDATE_ALL,
@@ -443,7 +486,8 @@ bucket_definitions:
443
486
  bucket: 'global[1]',
444
487
  checksum: 4,
445
488
  count: 4,
446
- priority: 3
489
+ priority: 3,
490
+ subscriptions: [{ default: 0 }]
447
491
  }
448
492
  ],
449
493
  last_op_id: '4',
@@ -479,32 +523,49 @@ bucket_definitions:
479
523
 
480
524
  const state = new BucketChecksumState({
481
525
  syncContext,
482
- syncParams: new RequestParameters({ sub: 'u1' }, {}),
483
- syncRules: SYNC_RULES_DYNAMIC,
526
+ tokenPayload: { sub: 'u1' },
527
+ syncRequest,
528
+ syncRules: {
529
+ syncRules: SYNC_RULES_DYNAMIC,
530
+ version: 1
531
+ },
484
532
  bucketStorage: storage
485
533
  });
486
534
 
487
- storage.getParameterSets = async (
488
- checkpoint: InternalOpId,
489
- lookups: ParameterLookup[]
490
- ): Promise<SqliteJsonRow[]> => {
491
- expect(checkpoint).toEqual(1n);
492
- expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
493
- return [{ id: 1 }, { id: 2 }];
494
- };
495
-
496
535
  const line = (await state.buildNextCheckpointLine({
497
- base: { checkpoint: 1n, lsn: '1' },
536
+ base: storage.makeCheckpoint(1n, (lookups) => {
537
+ expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
538
+ return [{ id: 1 }, { id: 2 }];
539
+ }),
498
540
  writeCheckpoint: null,
499
541
  update: CHECKPOINT_INVALIDATE_ALL
500
542
  }))!;
501
543
  expect(line.checkpointLine).toEqual({
502
544
  checkpoint: {
503
545
  buckets: [
504
- { bucket: 'by_project[1]', checksum: 1, count: 1, priority: 3 },
505
- { bucket: 'by_project[2]', checksum: 1, count: 1, priority: 3 }
546
+ {
547
+ bucket: 'by_project[1]',
548
+ checksum: 1,
549
+ count: 1,
550
+ priority: 3,
551
+ subscriptions: [{ default: 0 }]
552
+ },
553
+ {
554
+ bucket: 'by_project[2]',
555
+ checksum: 1,
556
+ count: 1,
557
+ priority: 3,
558
+ subscriptions: [{ default: 0 }]
559
+ }
506
560
  ],
507
561
  last_op_id: '1',
562
+ streams: [
563
+ {
564
+ is_default: true,
565
+ name: 'by_project',
566
+ errors: []
567
+ }
568
+ ],
508
569
  write_checkpoint: undefined
509
570
  }
510
571
  });
@@ -531,18 +592,12 @@ bucket_definitions:
531
592
  line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
532
593
  line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
533
594
 
534
- storage.getParameterSets = async (
535
- checkpoint: InternalOpId,
536
- lookups: ParameterLookup[]
537
- ): Promise<SqliteJsonRow[]> => {
538
- expect(checkpoint).toEqual(2n);
539
- expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
540
- return [{ id: 1 }, { id: 2 }, { id: 3 }];
541
- };
542
-
543
595
  // Now we get a new line
544
596
  const line2 = (await state.buildNextCheckpointLine({
545
- base: { checkpoint: 2n, lsn: '2' },
597
+ base: storage.makeCheckpoint(2n, (lookups) => {
598
+ expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
599
+ return [{ id: 1 }, { id: 2 }, { id: 3 }];
600
+ }),
546
601
  writeCheckpoint: null,
547
602
  update: {
548
603
  invalidateDataBuckets: false,
@@ -555,13 +610,245 @@ bucket_definitions:
555
610
  expect(line2.checkpointLine).toEqual({
556
611
  checkpoint_diff: {
557
612
  removed_buckets: [],
558
- updated_buckets: [{ bucket: 'by_project[3]', checksum: 1, count: 1, priority: 3 }],
613
+ updated_buckets: [
614
+ {
615
+ bucket: 'by_project[3]',
616
+ checksum: 1,
617
+ count: 1,
618
+ priority: 3,
619
+ subscriptions: [{ default: 0 }]
620
+ }
621
+ ],
559
622
  last_op_id: '2',
560
623
  write_checkpoint: undefined
561
624
  }
562
625
  });
563
626
  expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
564
627
  });
628
+
629
+ describe('streams', () => {
630
+ let source: { -readonly [P in keyof BucketSource]: BucketSource[P] };
631
+ let storage: MockBucketChecksumStateStorage;
632
+
633
+ function checksumState(source: string | boolean, options?: Partial<BucketChecksumStateOptions>) {
634
+ if (typeof source == 'boolean') {
635
+ source = `
636
+ streams:
637
+ stream:
638
+ auto_subscribe: ${source}
639
+ query: SELECT * FROM assets WHERE id IN ifnull(subscription.parameter('ids'), '["default"]');
640
+
641
+ config:
642
+ edition: 2
643
+ `;
644
+ }
645
+
646
+ const rules = SqlSyncRules.fromYaml(source, {
647
+ defaultSchema: 'public'
648
+ });
649
+
650
+ return new BucketChecksumState({
651
+ syncContext,
652
+ syncRequest,
653
+ tokenPayload,
654
+ syncRules: { syncRules: rules, version: 1 },
655
+ bucketStorage: storage,
656
+ ...options
657
+ });
658
+ }
659
+
660
+ beforeEach(() => {
661
+ storage = new MockBucketChecksumStateStorage();
662
+ storage.updateTestChecksum({ bucket: '1#stream|0["default"]', checksum: 1, count: 1 });
663
+ storage.updateTestChecksum({ bucket: '1#stream|0["a"]', checksum: 1, count: 1 });
664
+ storage.updateTestChecksum({ bucket: '1#stream|0["b"]', checksum: 1, count: 1 });
665
+ });
666
+
667
+ test('includes defaults', async () => {
668
+ const state = checksumState(true);
669
+ const line = await state.buildNextCheckpointLine({
670
+ base: storage.makeCheckpoint(1n),
671
+ writeCheckpoint: null,
672
+ update: CHECKPOINT_INVALIDATE_ALL
673
+ })!;
674
+ line?.advance();
675
+ expect(line?.checkpointLine).toEqual({
676
+ checkpoint: {
677
+ buckets: [
678
+ { bucket: '1#stream|0["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
679
+ ],
680
+ last_op_id: '1',
681
+ write_checkpoint: undefined,
682
+ streams: [{ name: 'stream', is_default: true, errors: [] }]
683
+ }
684
+ });
685
+ });
686
+
687
+ test('can exclude defaults', async () => {
688
+ const state = checksumState(true, { syncRequest: { streams: { include_defaults: false, subscriptions: [] } } });
689
+
690
+ const line = await state.buildNextCheckpointLine({
691
+ base: storage.makeCheckpoint(1n),
692
+ writeCheckpoint: null,
693
+ update: CHECKPOINT_INVALIDATE_ALL
694
+ })!;
695
+ line?.advance();
696
+ expect(line?.checkpointLine).toEqual({
697
+ checkpoint: {
698
+ buckets: [],
699
+ last_op_id: '1',
700
+ write_checkpoint: undefined,
701
+ streams: []
702
+ }
703
+ });
704
+ });
705
+
706
+ test('custom subscriptions', async () => {
707
+ const state = checksumState(true, {
708
+ syncRequest: {
709
+ streams: {
710
+ subscriptions: [
711
+ { stream: 'stream', parameters: { ids: '["a"]' }, override_priority: null },
712
+ { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 }
713
+ ]
714
+ }
715
+ }
716
+ });
717
+
718
+ const line = await state.buildNextCheckpointLine({
719
+ base: storage.makeCheckpoint(1n),
720
+ writeCheckpoint: null,
721
+ update: CHECKPOINT_INVALIDATE_ALL
722
+ })!;
723
+ line?.advance();
724
+ expect(line?.checkpointLine).toEqual({
725
+ checkpoint: {
726
+ buckets: [
727
+ { bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
728
+ { bucket: '1#stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 1 }] },
729
+ { bucket: '1#stream|0["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
730
+ ],
731
+ last_op_id: '1',
732
+ write_checkpoint: undefined,
733
+ streams: [{ name: 'stream', is_default: true, errors: [] }]
734
+ }
735
+ });
736
+ });
737
+
738
+ test('overlap between custom subscriptions', async () => {
739
+ const state = checksumState(false, {
740
+ syncRequest: {
741
+ streams: {
742
+ subscriptions: [
743
+ { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: null },
744
+ { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 }
745
+ ]
746
+ }
747
+ }
748
+ });
749
+
750
+ const line = await state.buildNextCheckpointLine({
751
+ base: storage.makeCheckpoint(1n),
752
+ writeCheckpoint: null,
753
+ update: CHECKPOINT_INVALIDATE_ALL
754
+ })!;
755
+ line?.advance();
756
+ expect(line?.checkpointLine).toEqual({
757
+ checkpoint: {
758
+ buckets: [
759
+ { bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
760
+ { bucket: '1#stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }, { sub: 1 }] }
761
+ ],
762
+ last_op_id: '1',
763
+ write_checkpoint: undefined,
764
+ streams: [{ name: 'stream', is_default: false, errors: [] }]
765
+ }
766
+ });
767
+ });
768
+
769
+ test('overlap between default and custom subscription', async () => {
770
+ const state = checksumState(true, {
771
+ syncRequest: {
772
+ streams: {
773
+ subscriptions: [{ stream: 'stream', parameters: { ids: '["a", "default"]' }, override_priority: 1 }]
774
+ }
775
+ }
776
+ });
777
+
778
+ const line = await state.buildNextCheckpointLine({
779
+ base: storage.makeCheckpoint(1n),
780
+ writeCheckpoint: null,
781
+ update: CHECKPOINT_INVALIDATE_ALL
782
+ })!;
783
+ line?.advance();
784
+ expect(line?.checkpointLine).toEqual({
785
+ checkpoint: {
786
+ buckets: [
787
+ { bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
788
+ {
789
+ bucket: '1#stream|0["default"]',
790
+ checksum: 1,
791
+ count: 1,
792
+ priority: 1,
793
+ subscriptions: [{ sub: 0 }, { default: 0 }]
794
+ }
795
+ ],
796
+ last_op_id: '1',
797
+ write_checkpoint: undefined,
798
+ streams: [{ name: 'stream', is_default: true, errors: [] }]
799
+ }
800
+ });
801
+ });
802
+
803
+ test('reports errors', async () => {
804
+ const state = checksumState(true, {
805
+ syncRequest: {
806
+ streams: {
807
+ subscriptions: [
808
+ { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: 1 },
809
+ { stream: 'stream', parameters: { ids: 'invalid json' }, override_priority: null }
810
+ ]
811
+ }
812
+ }
813
+ });
814
+
815
+ const line = await state.buildNextCheckpointLine({
816
+ base: storage.makeCheckpoint(1n),
817
+ writeCheckpoint: null,
818
+ update: CHECKPOINT_INVALIDATE_ALL
819
+ })!;
820
+ line?.advance();
821
+ expect(line?.checkpointLine).toEqual({
822
+ checkpoint: {
823
+ buckets: [
824
+ { bucket: '1#stream|0["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
825
+ { bucket: '1#stream|0["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
826
+ {
827
+ bucket: '1#stream|0["default"]',
828
+ checksum: 1,
829
+ count: 1,
830
+ priority: 3,
831
+ subscriptions: [{ default: 0 }]
832
+ }
833
+ ],
834
+ last_op_id: '1',
835
+ write_checkpoint: undefined,
836
+ streams: [
837
+ {
838
+ name: 'stream',
839
+ is_default: true,
840
+ errors: [
841
+ {
842
+ message: 'Error evaluating bucket ids: Unexpected token \'i\', "invalid json" is not valid JSON',
843
+ subscription: 1
844
+ }
845
+ ]
846
+ }
847
+ ]
848
+ }
849
+ });
850
+ });
851
+ });
565
852
  });
566
853
 
567
854
  class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
@@ -595,7 +882,19 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
595
882
  );
596
883
  }
597
884
 
598
- async getParameterSets(checkpoint: InternalOpId, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]> {
599
- throw new Error('Method not implemented.');
885
+ makeCheckpoint(
886
+ opId: InternalOpId,
887
+ parameters?: (lookups: ParameterLookup[]) => SqliteJsonRow[]
888
+ ): ReplicationCheckpoint {
889
+ return {
890
+ checkpoint: opId,
891
+ lsn: String(opId),
892
+ getParameterSets: async (lookups: ParameterLookup[]) => {
893
+ if (parameters == null) {
894
+ throw new Error(`getParametersSets not defined for checkpoint ${opId}`);
895
+ }
896
+ return parameters(lookups);
897
+ }
898
+ };
600
899
  }
601
900
  }