@powersync/service-core 1.19.2 → 1.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/dist/api/diagnostics.js +11 -4
  3. package/dist/api/diagnostics.js.map +1 -1
  4. package/dist/entry/commands/compact-action.js +13 -2
  5. package/dist/entry/commands/compact-action.js.map +1 -1
  6. package/dist/entry/commands/config-command.js +2 -2
  7. package/dist/entry/commands/config-command.js.map +1 -1
  8. package/dist/replication/AbstractReplicator.js +2 -5
  9. package/dist/replication/AbstractReplicator.js.map +1 -1
  10. package/dist/routes/configure-fastify.d.ts +84 -0
  11. package/dist/routes/endpoints/admin.d.ts +168 -0
  12. package/dist/routes/endpoints/admin.js +33 -20
  13. package/dist/routes/endpoints/admin.js.map +1 -1
  14. package/dist/routes/endpoints/sync-rules.js +6 -9
  15. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  16. package/dist/storage/BucketStorageFactory.d.ts +43 -15
  17. package/dist/storage/BucketStorageFactory.js +70 -1
  18. package/dist/storage/BucketStorageFactory.js.map +1 -1
  19. package/dist/storage/PersistedSyncRulesContent.d.ts +28 -2
  20. package/dist/storage/PersistedSyncRulesContent.js +79 -1
  21. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  22. package/dist/storage/StorageVersionConfig.d.ts +20 -0
  23. package/dist/storage/StorageVersionConfig.js +20 -0
  24. package/dist/storage/StorageVersionConfig.js.map +1 -0
  25. package/dist/storage/SyncRulesBucketStorage.d.ts +2 -1
  26. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  27. package/dist/storage/storage-index.d.ts +1 -0
  28. package/dist/storage/storage-index.js +1 -0
  29. package/dist/storage/storage-index.js.map +1 -1
  30. package/dist/sync/BucketChecksumState.d.ts +6 -2
  31. package/dist/sync/BucketChecksumState.js +85 -10
  32. package/dist/sync/BucketChecksumState.js.map +1 -1
  33. package/dist/util/config/collectors/config-collector.js +13 -0
  34. package/dist/util/config/collectors/config-collector.js.map +1 -1
  35. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +1 -1
  36. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +4 -4
  37. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  38. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +1 -1
  39. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +2 -2
  40. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  41. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +1 -1
  42. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +3 -3
  43. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
  44. package/dist/util/config/types.d.ts +1 -1
  45. package/dist/util/config/types.js.map +1 -1
  46. package/dist/util/env.d.ts +1 -0
  47. package/dist/util/env.js +5 -0
  48. package/dist/util/env.js.map +1 -1
  49. package/package.json +6 -6
  50. package/src/api/diagnostics.ts +12 -4
  51. package/src/entry/commands/compact-action.ts +15 -2
  52. package/src/entry/commands/config-command.ts +3 -3
  53. package/src/replication/AbstractReplicator.ts +3 -5
  54. package/src/routes/endpoints/admin.ts +42 -25
  55. package/src/routes/endpoints/sync-rules.ts +14 -13
  56. package/src/storage/BucketStorageFactory.ts +110 -19
  57. package/src/storage/PersistedSyncRulesContent.ts +114 -4
  58. package/src/storage/StorageVersionConfig.ts +30 -0
  59. package/src/storage/SyncRulesBucketStorage.ts +2 -1
  60. package/src/storage/storage-index.ts +1 -0
  61. package/src/sync/BucketChecksumState.ts +129 -16
  62. package/src/util/config/collectors/config-collector.ts +16 -0
  63. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +5 -5
  64. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +3 -3
  65. package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +4 -4
  66. package/src/util/config/types.ts +1 -2
  67. package/src/util/env.ts +5 -0
  68. package/test/src/config.test.ts +115 -0
  69. package/test/src/routes/admin.test.ts +48 -0
  70. package/test/src/routes/mocks.ts +22 -1
  71. package/test/src/routes/stream.test.ts +3 -2
  72. package/test/src/sync/BucketChecksumState.test.ts +285 -78
  73. package/tsconfig.tsbuildinfo +1 -1
@@ -13,8 +13,13 @@ import {
13
13
  WatchFilterEvent
14
14
  } from '@/index.js';
15
15
  import { JSONBig } from '@powersync/service-jsonbig';
16
- import { RequestJwtPayload, ScopedParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules';
17
- import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js';
16
+ import {
17
+ RequestJwtPayload,
18
+ ScopedParameterLookup,
19
+ SqliteJsonRow,
20
+ SqlSyncRules,
21
+ versionedHydrationState
22
+ } from '@powersync/service-sync-rules';
18
23
  import { beforeEach, describe, expect, test } from 'vitest';
19
24
 
20
25
  describe('BucketChecksumState', () => {
@@ -65,7 +70,7 @@ bucket_definitions:
65
70
  test('global bucket with update', async () => {
66
71
  const storage = new MockBucketChecksumStateStorage();
67
72
  // Set intial state
68
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
73
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
69
74
 
70
75
  const state = new BucketChecksumState({
71
76
  syncContext,
@@ -83,7 +88,7 @@ bucket_definitions:
83
88
  line.advance();
84
89
  expect(line.checkpointLine).toEqual({
85
90
  checkpoint: {
86
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
91
+ buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
87
92
  last_op_id: '1',
88
93
  write_checkpoint: undefined,
89
94
  streams: [{ name: 'global', is_default: true, errors: [] }]
@@ -91,26 +96,26 @@ bucket_definitions:
91
96
  });
92
97
  expect(line.bucketsToFetch).toEqual([
93
98
  {
94
- bucket: 'global[]',
99
+ bucket: '1#global[]',
95
100
  priority: 3
96
101
  }
97
102
  ]);
98
103
  // This is the bucket data to be fetched
99
- expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
104
+ expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
100
105
 
101
106
  // This similuates the bucket data being sent
102
107
  line.advance();
103
- line.updateBucketPosition({ bucket: 'global[]', nextAfter: 1n, hasMore: false });
108
+ line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: false });
104
109
 
105
110
  // Update bucket storage state
106
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 2, count: 2 });
111
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
107
112
 
108
113
  // Now we get a new line
109
114
  const line2 = (await state.buildNextCheckpointLine({
110
115
  base: storage.makeCheckpoint(2n),
111
116
  writeCheckpoint: null,
112
117
  update: {
113
- updatedDataBuckets: new Set(['global[]']),
118
+ updatedDataBuckets: new Set(['1#global[]']),
114
119
  invalidateDataBuckets: false,
115
120
  updatedParameterLookups: new Set(),
116
121
  invalidateParameterBuckets: false
@@ -120,12 +125,14 @@ bucket_definitions:
120
125
  expect(line2.checkpointLine).toEqual({
121
126
  checkpoint_diff: {
122
127
  removed_buckets: [],
123
- updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }],
128
+ updated_buckets: [
129
+ { bucket: '1#global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
130
+ ],
124
131
  last_op_id: '2',
125
132
  write_checkpoint: undefined
126
133
  }
127
134
  });
128
- expect(line2.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
135
+ expect(line2.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
129
136
  });
130
137
 
131
138
  test('global bucket with initial state', async () => {
@@ -134,13 +141,13 @@ bucket_definitions:
134
141
  /// (getFilteredBucketStates)
135
142
  const storage = new MockBucketChecksumStateStorage();
136
143
  // Set intial state
137
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
144
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
138
145
 
139
146
  const state = new BucketChecksumState({
140
147
  syncContext,
141
148
  tokenPayload,
142
149
  // Client sets the initial state here
143
- syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
150
+ syncRequest: { buckets: [{ name: '1#global[]', after: '1' }] },
144
151
  syncRules: SYNC_RULES_GLOBAL,
145
152
  bucketStorage: storage
146
153
  });
@@ -153,7 +160,7 @@ bucket_definitions:
153
160
  line.advance();
154
161
  expect(line.checkpointLine).toEqual({
155
162
  checkpoint: {
156
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
163
+ buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
157
164
  last_op_id: '1',
158
165
  write_checkpoint: undefined,
159
166
  streams: [{ name: 'global', is_default: true, errors: [] }]
@@ -161,19 +168,19 @@ bucket_definitions:
161
168
  });
162
169
  expect(line.bucketsToFetch).toEqual([
163
170
  {
164
- bucket: 'global[]',
171
+ bucket: '1#global[]',
165
172
  priority: 3
166
173
  }
167
174
  ]);
168
175
  // This is the main difference between this and the previous test
169
- expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
176
+ expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
170
177
  });
171
178
 
172
179
  test('multiple static buckets', async () => {
173
180
  const storage = new MockBucketChecksumStateStorage();
174
181
  // Set intial state
175
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
176
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
182
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
183
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
177
184
 
178
185
  const state = new BucketChecksumState({
179
186
  syncContext,
@@ -191,8 +198,8 @@ bucket_definitions:
191
198
  expect(line.checkpointLine).toEqual({
192
199
  checkpoint: {
193
200
  buckets: [
194
- { bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
195
- { bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
201
+ { bucket: '2#global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
202
+ { bucket: '2#global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
196
203
  ],
197
204
  last_op_id: '1',
198
205
  write_checkpoint: undefined,
@@ -201,25 +208,25 @@ bucket_definitions:
201
208
  });
202
209
  expect(line.bucketsToFetch).toEqual([
203
210
  {
204
- bucket: 'global[1]',
211
+ bucket: '2#global[1]',
205
212
  priority: 3
206
213
  },
207
214
  {
208
- bucket: 'global[2]',
215
+ bucket: '2#global[2]',
209
216
  priority: 3
210
217
  }
211
218
  ]);
212
219
  line.advance();
213
220
 
214
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
215
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
221
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
222
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
216
223
 
217
224
  const line2 = (await state.buildNextCheckpointLine({
218
225
  base: storage.makeCheckpoint(2n),
219
226
  writeCheckpoint: null,
220
227
  update: {
221
228
  ...CHECKPOINT_INVALIDATE_ALL,
222
- updatedDataBuckets: new Set(['global[1]', 'global[2]']),
229
+ updatedDataBuckets: new Set(['2#global[1]', '2#global[2]']),
223
230
  invalidateDataBuckets: false
224
231
  }
225
232
  }))!;
@@ -227,8 +234,8 @@ bucket_definitions:
227
234
  checkpoint_diff: {
228
235
  removed_buckets: [],
229
236
  updated_buckets: [
230
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
231
- { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
237
+ { bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
238
+ { bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
232
239
  ],
233
240
  last_op_id: '2',
234
241
  write_checkpoint: undefined
@@ -251,7 +258,7 @@ bucket_definitions:
251
258
  bucketStorage: storage
252
259
  });
253
260
 
254
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
261
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
255
262
 
256
263
  const line = (await state.buildNextCheckpointLine({
257
264
  base: storage.makeCheckpoint(1n),
@@ -261,7 +268,7 @@ bucket_definitions:
261
268
  line.advance();
262
269
  expect(line.checkpointLine).toEqual({
263
270
  checkpoint: {
264
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
271
+ buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
265
272
  last_op_id: '1',
266
273
  write_checkpoint: undefined,
267
274
  streams: [{ name: 'global', is_default: true, errors: [] }]
@@ -269,11 +276,11 @@ bucket_definitions:
269
276
  });
270
277
  expect(line.bucketsToFetch).toEqual([
271
278
  {
272
- bucket: 'global[]',
279
+ bucket: '1#global[]',
273
280
  priority: 3
274
281
  }
275
282
  ]);
276
- expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
283
+ expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
277
284
  });
278
285
 
279
286
  test('invalidating individual bucket', async () => {
@@ -281,8 +288,8 @@ bucket_definitions:
281
288
 
282
289
  const storage = new MockBucketChecksumStateStorage();
283
290
  // Set initial state
284
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
285
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
291
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
292
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
286
293
 
287
294
  const state = new BucketChecksumState({
288
295
  syncContext,
@@ -301,11 +308,11 @@ bucket_definitions:
301
308
  update: CHECKPOINT_INVALIDATE_ALL
302
309
  });
303
310
  line!.advance();
304
- line!.updateBucketPosition({ bucket: 'global[1]', nextAfter: 1n, hasMore: false });
305
- line!.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: false });
311
+ line!.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 1n, hasMore: false });
312
+ line!.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: false });
306
313
 
307
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
308
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
314
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
315
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
309
316
 
310
317
  const line2 = (await state.buildNextCheckpointLine({
311
318
  base: storage.makeCheckpoint(2n),
@@ -315,7 +322,7 @@ bucket_definitions:
315
322
  // Invalidate the state for global[1] - will only re-check the single bucket.
316
323
  // This is essentially inconsistent state, but is the simplest way to test that
317
324
  // the filter is working.
318
- updatedDataBuckets: new Set(['global[1]']),
325
+ updatedDataBuckets: new Set(['2#global[1]']),
319
326
  invalidateDataBuckets: false
320
327
  }
321
328
  }))!;
@@ -324,13 +331,13 @@ bucket_definitions:
324
331
  removed_buckets: [],
325
332
  updated_buckets: [
326
333
  // This does not include global[2], since it was not invalidated.
327
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
334
+ { bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
328
335
  ],
329
336
  last_op_id: '2',
330
337
  write_checkpoint: undefined
331
338
  }
332
339
  });
333
- expect(line2.bucketsToFetch).toEqual([{ bucket: 'global[1]', priority: 3 }]);
340
+ expect(line2.bucketsToFetch).toEqual([{ bucket: '2#global[1]', priority: 3 }]);
334
341
  });
335
342
 
336
343
  test('invalidating all buckets', async () => {
@@ -349,8 +356,8 @@ bucket_definitions:
349
356
  // storage.filter = state.checkpointFilter;
350
357
 
351
358
  // Set initial state
352
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
353
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
359
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
360
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
354
361
 
355
362
  const line = await state.buildNextCheckpointLine({
356
363
  base: storage.makeCheckpoint(1n),
@@ -360,8 +367,8 @@ bucket_definitions:
360
367
 
361
368
  line!.advance();
362
369
 
363
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
364
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
370
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
371
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
365
372
 
366
373
  const line2 = (await state.buildNextCheckpointLine({
367
374
  base: storage.makeCheckpoint(2n),
@@ -373,24 +380,24 @@ bucket_definitions:
373
380
  checkpoint_diff: {
374
381
  removed_buckets: [],
375
382
  updated_buckets: [
376
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
377
- { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
383
+ { bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
384
+ { bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
378
385
  ],
379
386
  last_op_id: '2',
380
387
  write_checkpoint: undefined
381
388
  }
382
389
  });
383
390
  expect(line2.bucketsToFetch).toEqual([
384
- { bucket: 'global[1]', priority: 3 },
385
- { bucket: 'global[2]', priority: 3 }
391
+ { bucket: '2#global[1]', priority: 3 },
392
+ { bucket: '2#global[2]', priority: 3 }
386
393
  ]);
387
394
  });
388
395
 
389
396
  test('interrupt and resume static buckets checkpoint', async () => {
390
397
  const storage = new MockBucketChecksumStateStorage();
391
398
  // Set intial state
392
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 3, count: 3 });
393
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 3, count: 3 });
399
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 3, count: 3 });
400
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 3, count: 3 });
394
401
 
395
402
  const state = new BucketChecksumState({
396
403
  syncContext,
@@ -409,8 +416,8 @@ bucket_definitions:
409
416
  expect(line.checkpointLine).toEqual({
410
417
  checkpoint: {
411
418
  buckets: [
412
- { bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
413
- { bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
419
+ { bucket: '2#global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
420
+ { bucket: '2#global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
414
421
  ],
415
422
  last_op_id: '3',
416
423
  write_checkpoint: undefined,
@@ -419,11 +426,11 @@ bucket_definitions:
419
426
  });
420
427
  expect(line.bucketsToFetch).toEqual([
421
428
  {
422
- bucket: 'global[1]',
429
+ bucket: '2#global[1]',
423
430
  priority: 3
424
431
  },
425
432
  {
426
- bucket: 'global[2]',
433
+ bucket: '2#global[2]',
427
434
  priority: 3
428
435
  }
429
436
  ]);
@@ -431,17 +438,17 @@ bucket_definitions:
431
438
  // This is the bucket data to be fetched
432
439
  expect(line.getFilteredBucketPositions()).toEqual(
433
440
  new Map([
434
- ['global[1]', 0n],
435
- ['global[2]', 0n]
441
+ ['2#global[1]', 0n],
442
+ ['2#global[2]', 0n]
436
443
  ])
437
444
  );
438
445
 
439
446
  // No data changes here.
440
447
  // We simulate partial data sent, before a checkpoint is interrupted.
441
448
  line.advance();
442
- line.updateBucketPosition({ bucket: 'global[1]', nextAfter: 3n, hasMore: false });
443
- line.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: true });
444
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
449
+ line.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 3n, hasMore: false });
450
+ line.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: true });
451
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 4, count: 4 });
445
452
 
446
453
  const line2 = (await state.buildNextCheckpointLine({
447
454
  base: storage.makeCheckpoint(4n),
@@ -449,7 +456,7 @@ bucket_definitions:
449
456
  update: {
450
457
  ...CHECKPOINT_INVALIDATE_ALL,
451
458
  invalidateDataBuckets: false,
452
- updatedDataBuckets: new Set(['global[1]'])
459
+ updatedDataBuckets: new Set(['2#global[1]'])
453
460
  }
454
461
  }))!;
455
462
  line2.advance();
@@ -458,7 +465,7 @@ bucket_definitions:
458
465
  removed_buckets: [],
459
466
  updated_buckets: [
460
467
  {
461
- bucket: 'global[1]',
468
+ bucket: '2#global[1]',
462
469
  checksum: 4,
463
470
  count: 4,
464
471
  priority: 3,
@@ -472,19 +479,19 @@ bucket_definitions:
472
479
  // This should contain both buckets, even though only one changed.
473
480
  expect(line2.bucketsToFetch).toEqual([
474
481
  {
475
- bucket: 'global[1]',
482
+ bucket: '2#global[1]',
476
483
  priority: 3
477
484
  },
478
485
  {
479
- bucket: 'global[2]',
486
+ bucket: '2#global[2]',
480
487
  priority: 3
481
488
  }
482
489
  ]);
483
490
 
484
491
  expect(line2.getFilteredBucketPositions()).toEqual(
485
492
  new Map([
486
- ['global[1]', 3n],
487
- ['global[2]', 1n]
493
+ ['2#global[1]', 3n],
494
+ ['2#global[2]', 1n]
488
495
  ])
489
496
  );
490
497
  });
@@ -492,9 +499,9 @@ bucket_definitions:
492
499
  test('dynamic buckets with updates', async () => {
493
500
  const storage = new MockBucketChecksumStateStorage();
494
501
  // Set intial state
495
- storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
496
- storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
497
- storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
502
+ storage.updateTestChecksum({ bucket: '3#by_project[1]', checksum: 1, count: 1 });
503
+ storage.updateTestChecksum({ bucket: '3#by_project[2]', checksum: 1, count: 1 });
504
+ storage.updateTestChecksum({ bucket: '3#by_project[3]', checksum: 1, count: 1 });
498
505
 
499
506
  const state = new BucketChecksumState({
500
507
  syncContext,
@@ -516,14 +523,14 @@ bucket_definitions:
516
523
  checkpoint: {
517
524
  buckets: [
518
525
  {
519
- bucket: 'by_project[1]',
526
+ bucket: '3#by_project[1]',
520
527
  checksum: 1,
521
528
  count: 1,
522
529
  priority: 3,
523
530
  subscriptions: [{ default: 0 }]
524
531
  },
525
532
  {
526
- bucket: 'by_project[2]',
533
+ bucket: '3#by_project[2]',
527
534
  checksum: 1,
528
535
  count: 1,
529
536
  priority: 3,
@@ -543,11 +550,11 @@ bucket_definitions:
543
550
  });
544
551
  expect(line.bucketsToFetch).toEqual([
545
552
  {
546
- bucket: 'by_project[1]',
553
+ bucket: '3#by_project[1]',
547
554
  priority: 3
548
555
  },
549
556
  {
550
- bucket: 'by_project[2]',
557
+ bucket: '3#by_project[2]',
551
558
  priority: 3
552
559
  }
553
560
  ]);
@@ -555,14 +562,14 @@ bucket_definitions:
555
562
  // This is the bucket data to be fetched
556
563
  expect(line.getFilteredBucketPositions()).toEqual(
557
564
  new Map([
558
- ['by_project[1]', 0n],
559
- ['by_project[2]', 0n]
565
+ ['3#by_project[1]', 0n],
566
+ ['3#by_project[2]', 0n]
560
567
  ])
561
568
  );
562
569
 
563
570
  line.advance();
564
- line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
565
- line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
571
+ line.updateBucketPosition({ bucket: '3#by_project[1]', nextAfter: 1n, hasMore: false });
572
+ line.updateBucketPosition({ bucket: '3#by_project[2]', nextAfter: 1n, hasMore: false });
566
573
 
567
574
  // Now we get a new line
568
575
  const line2 = (await state.buildNextCheckpointLine({
@@ -584,7 +591,7 @@ bucket_definitions:
584
591
  removed_buckets: [],
585
592
  updated_buckets: [
586
593
  {
587
- bucket: 'by_project[3]',
594
+ bucket: '3#by_project[3]',
588
595
  checksum: 1,
589
596
  count: 1,
590
597
  priority: 3,
@@ -595,7 +602,7 @@ bucket_definitions:
595
602
  write_checkpoint: undefined
596
603
  }
597
604
  });
598
- expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
605
+ expect(line2.getFilteredBucketPositions()).toEqual(new Map([['3#by_project[3]', 0n]]));
599
606
  });
600
607
 
601
608
  describe('streams', () => {
@@ -820,6 +827,206 @@ config:
820
827
  });
821
828
  });
822
829
  });
830
+
831
+ describe('parameter query result logging', () => {
832
+ test('logs parameter query results for dynamic buckets', async () => {
833
+ const storage = new MockBucketChecksumStateStorage();
834
+ storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
835
+ storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
836
+ storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
837
+
838
+ const logMessages: string[] = [];
839
+ const logData: any[] = [];
840
+ const mockLogger = {
841
+ info: (message: string, data: any) => {
842
+ logMessages.push(message);
843
+ logData.push(data);
844
+ },
845
+ error: () => {},
846
+ warn: () => {},
847
+ debug: () => {}
848
+ };
849
+
850
+ const state = new BucketChecksumState({
851
+ syncContext,
852
+ tokenPayload: new JwtPayload({ sub: 'u1' }),
853
+ syncRequest,
854
+ syncRules: SYNC_RULES_DYNAMIC,
855
+ bucketStorage: storage,
856
+ logger: mockLogger as any
857
+ });
858
+
859
+ const line = (await state.buildNextCheckpointLine({
860
+ base: storage.makeCheckpoint(1n, (lookups) => {
861
+ return [{ id: 1 }, { id: 2 }, { id: 3 }];
862
+ }),
863
+ writeCheckpoint: null,
864
+ update: CHECKPOINT_INVALIDATE_ALL
865
+ }))!;
866
+ line.advance();
867
+
868
+ expect(logMessages[0]).toContain('param_results: 3');
869
+ expect(logData[0].parameter_query_results).toBe(3);
870
+ });
871
+
872
+ test('throws error with breakdown when parameter query limit is exceeded', async () => {
873
+ const SYNC_RULES_MULTI = SqlSyncRules.fromYaml(
874
+ `
875
+ bucket_definitions:
876
+ projects:
877
+ parameters: select id from projects where user_id = request.user_id()
878
+ data: []
879
+ tasks:
880
+ parameters: select id from tasks where user_id = request.user_id()
881
+ data: []
882
+ comments:
883
+ parameters: select id from comments where user_id = request.user_id()
884
+ data: []
885
+ `,
886
+ { defaultSchema: 'public' }
887
+ ).config.hydrate({ hydrationState: versionedHydrationState(4) });
888
+
889
+ const storage = new MockBucketChecksumStateStorage();
890
+
891
+ const errorMessages: string[] = [];
892
+ const errorData: any[] = [];
893
+ const mockLogger = {
894
+ info: () => {},
895
+ error: (message: string, data: any) => {
896
+ errorMessages.push(message);
897
+ errorData.push(data);
898
+ },
899
+ warn: () => {},
900
+ debug: () => {}
901
+ };
902
+
903
+ const smallContext = new SyncContext({
904
+ maxBuckets: 100,
905
+ maxParameterQueryResults: 50,
906
+ maxDataFetchConcurrency: 10
907
+ });
908
+
909
+ const state = new BucketChecksumState({
910
+ syncContext: smallContext,
911
+ tokenPayload: new JwtPayload({ sub: 'u1' }),
912
+ syncRequest,
913
+ syncRules: SYNC_RULES_MULTI,
914
+ bucketStorage: storage,
915
+ logger: mockLogger as any
916
+ });
917
+
918
+ // Create 60 total results: 30 projects + 20 tasks + 10 comments
919
+ const projectIds = Array.from({ length: 30 }, (_, i) => ({ id: i + 1 }));
920
+ const taskIds = Array.from({ length: 20 }, (_, i) => ({ id: i + 1 }));
921
+ const commentIds = Array.from({ length: 10 }, (_, i) => ({ id: i + 1 }));
922
+
923
+ for (let i = 1; i <= 30; i++) {
924
+ storage.updateTestChecksum({ bucket: `projects[${i}]`, checksum: 1, count: 1 });
925
+ }
926
+ for (let i = 1; i <= 20; i++) {
927
+ storage.updateTestChecksum({ bucket: `tasks[${i}]`, checksum: 1, count: 1 });
928
+ }
929
+ for (let i = 1; i <= 10; i++) {
930
+ storage.updateTestChecksum({ bucket: `comments[${i}]`, checksum: 1, count: 1 });
931
+ }
932
+
933
+ await expect(
934
+ state.buildNextCheckpointLine({
935
+ base: storage.makeCheckpoint(1n, (lookups) => {
936
+ const lookup = lookups[0];
937
+ const lookupName = lookup.values[0];
938
+ if (lookupName === 'projects') {
939
+ return projectIds;
940
+ } else if (lookupName === 'tasks') {
941
+ return taskIds;
942
+ } else {
943
+ return commentIds;
944
+ }
945
+ }),
946
+ writeCheckpoint: null,
947
+ update: CHECKPOINT_INVALIDATE_ALL
948
+ })
949
+ ).rejects.toThrow('Too many parameter query results: 60 (limit of 50)');
950
+
951
+ // Verify error log includes breakdown
952
+ expect(errorMessages[0]).toContain('Parameter query results by definition:');
953
+ expect(errorMessages[0]).toContain('projects: 30');
954
+ expect(errorMessages[0]).toContain('tasks: 20');
955
+ expect(errorMessages[0]).toContain('comments: 10');
956
+
957
+ expect(errorData[0].parameter_query_results).toBe(60);
958
+ expect(errorData[0].parameter_query_results_by_definition).toEqual({
959
+ projects: 30,
960
+ tasks: 20,
961
+ comments: 10
962
+ });
963
+ });
964
+
965
+ test('limits breakdown to top 10 definitions', async () => {
966
+ // Create sync rules with 15 different definitions with dynamic parameters
967
+ let yamlDefinitions = 'bucket_definitions:\n';
968
+ for (let i = 1; i <= 15; i++) {
969
+ yamlDefinitions += ` def${i}:\n`;
970
+ yamlDefinitions += ` parameters: select id from def${i}_table where user_id = request.user_id()\n`;
971
+ yamlDefinitions += ` data: []\n`;
972
+ }
973
+
974
+ const SYNC_RULES_MANY = SqlSyncRules.fromYaml(yamlDefinitions, { defaultSchema: 'public' }).config.hydrate({
975
+ hydrationState: versionedHydrationState(5)
976
+ });
977
+
978
+ const storage = new MockBucketChecksumStateStorage();
979
+
980
+ const errorMessages: string[] = [];
981
+ const mockLogger = {
982
+ info: () => {},
983
+ error: (message: string) => {
984
+ errorMessages.push(message);
985
+ },
986
+ warn: () => {},
987
+ debug: () => {}
988
+ };
989
+
990
+ const smallContext = new SyncContext({
991
+ maxBuckets: 100,
992
+ maxParameterQueryResults: 10,
993
+ maxDataFetchConcurrency: 10
994
+ });
995
+
996
+ const state = new BucketChecksumState({
997
+ syncContext: smallContext,
998
+ tokenPayload: new JwtPayload({ sub: 'u1' }),
999
+ syncRequest,
1000
+ syncRules: SYNC_RULES_MANY,
1001
+ bucketStorage: storage,
1002
+ logger: mockLogger as any
1003
+ });
1004
+
1005
+ // Each definition creates one bucket, total 15 buckets
1006
+ for (let i = 1; i <= 15; i++) {
1007
+ storage.updateTestChecksum({ bucket: `def${i}[${i}]`, checksum: 1, count: 1 });
1008
+ }
1009
+
1010
+ await expect(
1011
+ state.buildNextCheckpointLine({
1012
+ base: storage.makeCheckpoint(1n, (lookups) => {
1013
+ // Return one result for each definition
1014
+ return [{ id: 1 }];
1015
+ }),
1016
+ writeCheckpoint: null,
1017
+ update: CHECKPOINT_INVALIDATE_ALL
1018
+ })
1019
+ ).rejects.toThrow('Too many parameter query results: 15 (limit of 10)');
1020
+
1021
+ // Verify only top 10 are shown
1022
+ const errorMessage = errorMessages[0];
1023
+ expect(errorMessage).toContain('... and 5 more results from 5 definitions');
1024
+
1025
+ // Count how many definitions are listed (should be exactly 10)
1026
+ const defMatches = errorMessage.match(/def\d+:/g);
1027
+ expect(defMatches?.length).toBe(10);
1028
+ });
1029
+ });
823
1030
  });
824
1031
 
825
1032
  class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {