@powersync/service-core 1.19.2 → 1.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/dist/api/diagnostics.js +11 -4
  3. package/dist/api/diagnostics.js.map +1 -1
  4. package/dist/entry/commands/compact-action.js +13 -2
  5. package/dist/entry/commands/compact-action.js.map +1 -1
  6. package/dist/entry/commands/config-command.js +2 -2
  7. package/dist/entry/commands/config-command.js.map +1 -1
  8. package/dist/replication/AbstractReplicator.js +2 -5
  9. package/dist/replication/AbstractReplicator.js.map +1 -1
  10. package/dist/routes/configure-fastify.d.ts +84 -0
  11. package/dist/routes/endpoints/admin.d.ts +168 -0
  12. package/dist/routes/endpoints/admin.js +34 -20
  13. package/dist/routes/endpoints/admin.js.map +1 -1
  14. package/dist/routes/endpoints/sync-rules.js +6 -9
  15. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  16. package/dist/routes/endpoints/sync-stream.js +6 -1
  17. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  18. package/dist/storage/BucketStorageBatch.d.ts +21 -8
  19. package/dist/storage/BucketStorageBatch.js.map +1 -1
  20. package/dist/storage/BucketStorageFactory.d.ts +48 -15
  21. package/dist/storage/BucketStorageFactory.js +70 -1
  22. package/dist/storage/BucketStorageFactory.js.map +1 -1
  23. package/dist/storage/ChecksumCache.d.ts +5 -2
  24. package/dist/storage/ChecksumCache.js +8 -4
  25. package/dist/storage/ChecksumCache.js.map +1 -1
  26. package/dist/storage/PersistedSyncRulesContent.d.ts +33 -3
  27. package/dist/storage/PersistedSyncRulesContent.js +80 -1
  28. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  29. package/dist/storage/SourceTable.d.ts +7 -2
  30. package/dist/storage/SourceTable.js.map +1 -1
  31. package/dist/storage/StorageVersionConfig.d.ts +53 -0
  32. package/dist/storage/StorageVersionConfig.js +53 -0
  33. package/dist/storage/StorageVersionConfig.js.map +1 -0
  34. package/dist/storage/SyncRulesBucketStorage.d.ts +14 -4
  35. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  36. package/dist/storage/storage-index.d.ts +1 -0
  37. package/dist/storage/storage-index.js +1 -0
  38. package/dist/storage/storage-index.js.map +1 -1
  39. package/dist/sync/BucketChecksumState.d.ts +8 -4
  40. package/dist/sync/BucketChecksumState.js +97 -52
  41. package/dist/sync/BucketChecksumState.js.map +1 -1
  42. package/dist/sync/sync.js.map +1 -1
  43. package/dist/sync/util.d.ts +1 -0
  44. package/dist/sync/util.js +10 -0
  45. package/dist/sync/util.js.map +1 -1
  46. package/dist/util/config/collectors/config-collector.js +13 -0
  47. package/dist/util/config/collectors/config-collector.js.map +1 -1
  48. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +1 -1
  49. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +4 -4
  50. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  51. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +1 -1
  52. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +2 -2
  53. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  54. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +1 -1
  55. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +3 -3
  56. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
  57. package/dist/util/config/types.d.ts +1 -1
  58. package/dist/util/config/types.js.map +1 -1
  59. package/dist/util/env.d.ts +1 -0
  60. package/dist/util/env.js +5 -0
  61. package/dist/util/env.js.map +1 -1
  62. package/package.json +6 -6
  63. package/src/api/diagnostics.ts +12 -4
  64. package/src/entry/commands/compact-action.ts +15 -2
  65. package/src/entry/commands/config-command.ts +3 -3
  66. package/src/replication/AbstractReplicator.ts +3 -5
  67. package/src/routes/endpoints/admin.ts +43 -25
  68. package/src/routes/endpoints/sync-rules.ts +14 -13
  69. package/src/routes/endpoints/sync-stream.ts +6 -1
  70. package/src/storage/BucketStorageBatch.ts +23 -9
  71. package/src/storage/BucketStorageFactory.ts +116 -19
  72. package/src/storage/ChecksumCache.ts +14 -6
  73. package/src/storage/PersistedSyncRulesContent.ts +119 -4
  74. package/src/storage/SourceTable.ts +7 -1
  75. package/src/storage/StorageVersionConfig.ts +78 -0
  76. package/src/storage/SyncRulesBucketStorage.ts +20 -4
  77. package/src/storage/storage-index.ts +1 -0
  78. package/src/sync/BucketChecksumState.ts +147 -65
  79. package/src/sync/sync.ts +9 -3
  80. package/src/sync/util.ts +10 -0
  81. package/src/util/config/collectors/config-collector.ts +16 -0
  82. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +5 -5
  83. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +3 -3
  84. package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +4 -4
  85. package/src/util/config/types.ts +1 -2
  86. package/src/util/env.ts +5 -0
  87. package/test/src/checksum_cache.test.ts +102 -57
  88. package/test/src/config.test.ts +115 -0
  89. package/test/src/routes/admin.test.ts +48 -0
  90. package/test/src/routes/mocks.ts +22 -1
  91. package/test/src/routes/stream.test.ts +3 -2
  92. package/test/src/sync/BucketChecksumState.test.ts +332 -93
  93. package/test/src/utils.ts +9 -0
  94. package/tsconfig.tsbuildinfo +1 -1
@@ -13,11 +13,44 @@ import {
13
13
  WatchFilterEvent
14
14
  } from '@/index.js';
15
15
  import { JSONBig } from '@powersync/service-jsonbig';
16
- import { RequestJwtPayload, ScopedParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules';
17
- import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js';
16
+ import {
17
+ ParameterIndexLookupCreator,
18
+ RequestJwtPayload,
19
+ ScopedParameterLookup,
20
+ SqliteJsonRow,
21
+ SqliteRow,
22
+ SqlSyncRules,
23
+ TablePattern,
24
+ SourceTableInterface,
25
+ versionedHydrationState
26
+ } from '@powersync/service-sync-rules';
27
+ import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
18
28
  import { beforeEach, describe, expect, test } from 'vitest';
19
29
 
20
30
  describe('BucketChecksumState', () => {
31
+ const LOOKUP_SOURCE: ParameterIndexLookupCreator = {
32
+ get defaultLookupScope(): ParameterLookupScope {
33
+ return {
34
+ lookupName: 'lookup',
35
+ queryId: '0',
36
+ source: LOOKUP_SOURCE
37
+ };
38
+ },
39
+ getSourceTables(): Set<TablePattern> {
40
+ return new Set();
41
+ },
42
+ evaluateParameterRow(_sourceTable: SourceTableInterface, _row: SqliteRow) {
43
+ return [];
44
+ },
45
+ tableSyncsParameters(_table: SourceTableInterface): boolean {
46
+ return false;
47
+ }
48
+ };
49
+
50
+ function lookupScope(lookupName: string, queryId: string): ParameterLookupScope {
51
+ return { lookupName, queryId, source: LOOKUP_SOURCE };
52
+ }
53
+
21
54
  // Single global[] bucket.
22
55
  // We don't care about data in these tests
23
56
  const SYNC_RULES_GLOBAL = SqlSyncRules.fromYaml(
@@ -62,10 +95,14 @@ bucket_definitions:
62
95
  const syncRequest: StreamingSyncRequest = {};
63
96
  const tokenPayload = new JwtPayload({ sub: '' });
64
97
 
98
+ function bucketStarts(requests: { bucket: string; start: InternalOpId }[]) {
99
+ return new Map(requests.map((request) => [request.bucket, request.start]));
100
+ }
101
+
65
102
  test('global bucket with update', async () => {
66
103
  const storage = new MockBucketChecksumStateStorage();
67
104
  // Set intial state
68
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
105
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
69
106
 
70
107
  const state = new BucketChecksumState({
71
108
  syncContext,
@@ -83,34 +120,34 @@ bucket_definitions:
83
120
  line.advance();
84
121
  expect(line.checkpointLine).toEqual({
85
122
  checkpoint: {
86
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
123
+ buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
87
124
  last_op_id: '1',
88
125
  write_checkpoint: undefined,
89
126
  streams: [{ name: 'global', is_default: true, errors: [] }]
90
127
  }
91
128
  });
92
- expect(line.bucketsToFetch).toEqual([
129
+ expect(line.bucketsToFetch).toMatchObject([
93
130
  {
94
- bucket: 'global[]',
131
+ bucket: '1#global[]',
95
132
  priority: 3
96
133
  }
97
134
  ]);
98
135
  // This is the bucket data to be fetched
99
- expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
136
+ expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
100
137
 
101
138
  // This similuates the bucket data being sent
102
139
  line.advance();
103
- line.updateBucketPosition({ bucket: 'global[]', nextAfter: 1n, hasMore: false });
140
+ line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: false });
104
141
 
105
142
  // Update bucket storage state
106
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 2, count: 2 });
143
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
107
144
 
108
145
  // Now we get a new line
109
146
  const line2 = (await state.buildNextCheckpointLine({
110
147
  base: storage.makeCheckpoint(2n),
111
148
  writeCheckpoint: null,
112
149
  update: {
113
- updatedDataBuckets: new Set(['global[]']),
150
+ updatedDataBuckets: new Set(['1#global[]']),
114
151
  invalidateDataBuckets: false,
115
152
  updatedParameterLookups: new Set(),
116
153
  invalidateParameterBuckets: false
@@ -120,12 +157,14 @@ bucket_definitions:
120
157
  expect(line2.checkpointLine).toEqual({
121
158
  checkpoint_diff: {
122
159
  removed_buckets: [],
123
- updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }],
160
+ updated_buckets: [
161
+ { bucket: '1#global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
162
+ ],
124
163
  last_op_id: '2',
125
164
  write_checkpoint: undefined
126
165
  }
127
166
  });
128
- expect(line2.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
167
+ expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
129
168
  });
130
169
 
131
170
  test('global bucket with initial state', async () => {
@@ -134,13 +173,13 @@ bucket_definitions:
134
173
  /// (getFilteredBucketStates)
135
174
  const storage = new MockBucketChecksumStateStorage();
136
175
  // Set intial state
137
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
176
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
138
177
 
139
178
  const state = new BucketChecksumState({
140
179
  syncContext,
141
180
  tokenPayload,
142
181
  // Client sets the initial state here
143
- syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
182
+ syncRequest: { buckets: [{ name: '1#global[]', after: '1' }] },
144
183
  syncRules: SYNC_RULES_GLOBAL,
145
184
  bucketStorage: storage
146
185
  });
@@ -153,27 +192,27 @@ bucket_definitions:
153
192
  line.advance();
154
193
  expect(line.checkpointLine).toEqual({
155
194
  checkpoint: {
156
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
195
+ buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
157
196
  last_op_id: '1',
158
197
  write_checkpoint: undefined,
159
198
  streams: [{ name: 'global', is_default: true, errors: [] }]
160
199
  }
161
200
  });
162
- expect(line.bucketsToFetch).toEqual([
201
+ expect(line.bucketsToFetch).toMatchObject([
163
202
  {
164
- bucket: 'global[]',
203
+ bucket: '1#global[]',
165
204
  priority: 3
166
205
  }
167
206
  ]);
168
207
  // This is the main difference between this and the previous test
169
- expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
208
+ expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
170
209
  });
171
210
 
172
211
  test('multiple static buckets', async () => {
173
212
  const storage = new MockBucketChecksumStateStorage();
174
213
  // Set intial state
175
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
176
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
214
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
215
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
177
216
 
178
217
  const state = new BucketChecksumState({
179
218
  syncContext,
@@ -191,35 +230,35 @@ bucket_definitions:
191
230
  expect(line.checkpointLine).toEqual({
192
231
  checkpoint: {
193
232
  buckets: [
194
- { bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
195
- { bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
233
+ { bucket: '2#global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
234
+ { bucket: '2#global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
196
235
  ],
197
236
  last_op_id: '1',
198
237
  write_checkpoint: undefined,
199
238
  streams: [{ name: 'global', is_default: true, errors: [] }]
200
239
  }
201
240
  });
202
- expect(line.bucketsToFetch).toEqual([
241
+ expect(line.bucketsToFetch).toMatchObject([
203
242
  {
204
- bucket: 'global[1]',
243
+ bucket: '2#global[1]',
205
244
  priority: 3
206
245
  },
207
246
  {
208
- bucket: 'global[2]',
247
+ bucket: '2#global[2]',
209
248
  priority: 3
210
249
  }
211
250
  ]);
212
251
  line.advance();
213
252
 
214
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
215
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
253
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
254
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
216
255
 
217
256
  const line2 = (await state.buildNextCheckpointLine({
218
257
  base: storage.makeCheckpoint(2n),
219
258
  writeCheckpoint: null,
220
259
  update: {
221
260
  ...CHECKPOINT_INVALIDATE_ALL,
222
- updatedDataBuckets: new Set(['global[1]', 'global[2]']),
261
+ updatedDataBuckets: new Set(['2#global[1]', '2#global[2]']),
223
262
  invalidateDataBuckets: false
224
263
  }
225
264
  }))!;
@@ -227,8 +266,8 @@ bucket_definitions:
227
266
  checkpoint_diff: {
228
267
  removed_buckets: [],
229
268
  updated_buckets: [
230
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
231
- { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
269
+ { bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
270
+ { bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
232
271
  ],
233
272
  last_op_id: '2',
234
273
  write_checkpoint: undefined
@@ -251,7 +290,7 @@ bucket_definitions:
251
290
  bucketStorage: storage
252
291
  });
253
292
 
254
- storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
293
+ storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
255
294
 
256
295
  const line = (await state.buildNextCheckpointLine({
257
296
  base: storage.makeCheckpoint(1n),
@@ -261,19 +300,19 @@ bucket_definitions:
261
300
  line.advance();
262
301
  expect(line.checkpointLine).toEqual({
263
302
  checkpoint: {
264
- buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
303
+ buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
265
304
  last_op_id: '1',
266
305
  write_checkpoint: undefined,
267
306
  streams: [{ name: 'global', is_default: true, errors: [] }]
268
307
  }
269
308
  });
270
- expect(line.bucketsToFetch).toEqual([
309
+ expect(line.bucketsToFetch).toMatchObject([
271
310
  {
272
- bucket: 'global[]',
311
+ bucket: '1#global[]',
273
312
  priority: 3
274
313
  }
275
314
  ]);
276
- expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
315
+ expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
277
316
  });
278
317
 
279
318
  test('invalidating individual bucket', async () => {
@@ -281,8 +320,8 @@ bucket_definitions:
281
320
 
282
321
  const storage = new MockBucketChecksumStateStorage();
283
322
  // Set initial state
284
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
285
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
323
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
324
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
286
325
 
287
326
  const state = new BucketChecksumState({
288
327
  syncContext,
@@ -301,11 +340,11 @@ bucket_definitions:
301
340
  update: CHECKPOINT_INVALIDATE_ALL
302
341
  });
303
342
  line!.advance();
304
- line!.updateBucketPosition({ bucket: 'global[1]', nextAfter: 1n, hasMore: false });
305
- line!.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: false });
343
+ line!.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 1n, hasMore: false });
344
+ line!.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: false });
306
345
 
307
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
308
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
346
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
347
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
309
348
 
310
349
  const line2 = (await state.buildNextCheckpointLine({
311
350
  base: storage.makeCheckpoint(2n),
@@ -315,7 +354,7 @@ bucket_definitions:
315
354
  // Invalidate the state for global[1] - will only re-check the single bucket.
316
355
  // This is essentially inconsistent state, but is the simplest way to test that
317
356
  // the filter is working.
318
- updatedDataBuckets: new Set(['global[1]']),
357
+ updatedDataBuckets: new Set(['2#global[1]']),
319
358
  invalidateDataBuckets: false
320
359
  }
321
360
  }))!;
@@ -324,13 +363,13 @@ bucket_definitions:
324
363
  removed_buckets: [],
325
364
  updated_buckets: [
326
365
  // This does not include global[2], since it was not invalidated.
327
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
366
+ { bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
328
367
  ],
329
368
  last_op_id: '2',
330
369
  write_checkpoint: undefined
331
370
  }
332
371
  });
333
- expect(line2.bucketsToFetch).toEqual([{ bucket: 'global[1]', priority: 3 }]);
372
+ expect(line2.bucketsToFetch).toMatchObject([{ bucket: '2#global[1]', priority: 3 }]);
334
373
  });
335
374
 
336
375
  test('invalidating all buckets', async () => {
@@ -349,8 +388,8 @@ bucket_definitions:
349
388
  // storage.filter = state.checkpointFilter;
350
389
 
351
390
  // Set initial state
352
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
353
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
391
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
392
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
354
393
 
355
394
  const line = await state.buildNextCheckpointLine({
356
395
  base: storage.makeCheckpoint(1n),
@@ -360,8 +399,8 @@ bucket_definitions:
360
399
 
361
400
  line!.advance();
362
401
 
363
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
364
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
402
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
403
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
365
404
 
366
405
  const line2 = (await state.buildNextCheckpointLine({
367
406
  base: storage.makeCheckpoint(2n),
@@ -373,24 +412,24 @@ bucket_definitions:
373
412
  checkpoint_diff: {
374
413
  removed_buckets: [],
375
414
  updated_buckets: [
376
- { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
377
- { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
415
+ { bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
416
+ { bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
378
417
  ],
379
418
  last_op_id: '2',
380
419
  write_checkpoint: undefined
381
420
  }
382
421
  });
383
- expect(line2.bucketsToFetch).toEqual([
384
- { bucket: 'global[1]', priority: 3 },
385
- { bucket: 'global[2]', priority: 3 }
422
+ expect(line2.bucketsToFetch).toMatchObject([
423
+ { bucket: '2#global[1]', priority: 3 },
424
+ { bucket: '2#global[2]', priority: 3 }
386
425
  ]);
387
426
  });
388
427
 
389
428
  test('interrupt and resume static buckets checkpoint', async () => {
390
429
  const storage = new MockBucketChecksumStateStorage();
391
430
  // Set intial state
392
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 3, count: 3 });
393
- storage.updateTestChecksum({ bucket: 'global[2]', checksum: 3, count: 3 });
431
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 3, count: 3 });
432
+ storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 3, count: 3 });
394
433
 
395
434
  const state = new BucketChecksumState({
396
435
  syncContext,
@@ -409,39 +448,39 @@ bucket_definitions:
409
448
  expect(line.checkpointLine).toEqual({
410
449
  checkpoint: {
411
450
  buckets: [
412
- { bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
413
- { bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
451
+ { bucket: '2#global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
452
+ { bucket: '2#global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
414
453
  ],
415
454
  last_op_id: '3',
416
455
  write_checkpoint: undefined,
417
456
  streams: [{ name: 'global', is_default: true, errors: [] }]
418
457
  }
419
458
  });
420
- expect(line.bucketsToFetch).toEqual([
459
+ expect(line.bucketsToFetch).toMatchObject([
421
460
  {
422
- bucket: 'global[1]',
461
+ bucket: '2#global[1]',
423
462
  priority: 3
424
463
  },
425
464
  {
426
- bucket: 'global[2]',
465
+ bucket: '2#global[2]',
427
466
  priority: 3
428
467
  }
429
468
  ]);
430
469
 
431
470
  // This is the bucket data to be fetched
432
- expect(line.getFilteredBucketPositions()).toEqual(
471
+ expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(
433
472
  new Map([
434
- ['global[1]', 0n],
435
- ['global[2]', 0n]
473
+ ['2#global[1]', 0n],
474
+ ['2#global[2]', 0n]
436
475
  ])
437
476
  );
438
477
 
439
478
  // No data changes here.
440
479
  // We simulate partial data sent, before a checkpoint is interrupted.
441
480
  line.advance();
442
- line.updateBucketPosition({ bucket: 'global[1]', nextAfter: 3n, hasMore: false });
443
- line.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: true });
444
- storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
481
+ line.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 3n, hasMore: false });
482
+ line.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: true });
483
+ storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 4, count: 4 });
445
484
 
446
485
  const line2 = (await state.buildNextCheckpointLine({
447
486
  base: storage.makeCheckpoint(4n),
@@ -449,7 +488,7 @@ bucket_definitions:
449
488
  update: {
450
489
  ...CHECKPOINT_INVALIDATE_ALL,
451
490
  invalidateDataBuckets: false,
452
- updatedDataBuckets: new Set(['global[1]'])
491
+ updatedDataBuckets: new Set(['2#global[1]'])
453
492
  }
454
493
  }))!;
455
494
  line2.advance();
@@ -458,7 +497,7 @@ bucket_definitions:
458
497
  removed_buckets: [],
459
498
  updated_buckets: [
460
499
  {
461
- bucket: 'global[1]',
500
+ bucket: '2#global[1]',
462
501
  checksum: 4,
463
502
  count: 4,
464
503
  priority: 3,
@@ -470,21 +509,21 @@ bucket_definitions:
470
509
  }
471
510
  });
472
511
  // This should contain both buckets, even though only one changed.
473
- expect(line2.bucketsToFetch).toEqual([
512
+ expect(line2.bucketsToFetch).toMatchObject([
474
513
  {
475
- bucket: 'global[1]',
514
+ bucket: '2#global[1]',
476
515
  priority: 3
477
516
  },
478
517
  {
479
- bucket: 'global[2]',
518
+ bucket: '2#global[2]',
480
519
  priority: 3
481
520
  }
482
521
  ]);
483
522
 
484
- expect(line2.getFilteredBucketPositions()).toEqual(
523
+ expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(
485
524
  new Map([
486
- ['global[1]', 3n],
487
- ['global[2]', 1n]
525
+ ['2#global[1]', 3n],
526
+ ['2#global[2]', 1n]
488
527
  ])
489
528
  );
490
529
  });
@@ -492,9 +531,9 @@ bucket_definitions:
492
531
  test('dynamic buckets with updates', async () => {
493
532
  const storage = new MockBucketChecksumStateStorage();
494
533
  // Set intial state
495
- storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
496
- storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
497
- storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
534
+ storage.updateTestChecksum({ bucket: '3#by_project[1]', checksum: 1, count: 1 });
535
+ storage.updateTestChecksum({ bucket: '3#by_project[2]', checksum: 1, count: 1 });
536
+ storage.updateTestChecksum({ bucket: '3#by_project[3]', checksum: 1, count: 1 });
498
537
 
499
538
  const state = new BucketChecksumState({
500
539
  syncContext,
@@ -506,7 +545,7 @@ bucket_definitions:
506
545
 
507
546
  const line = (await state.buildNextCheckpointLine({
508
547
  base: storage.makeCheckpoint(1n, (lookups) => {
509
- expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]);
548
+ expect(lookups).toEqual([ScopedParameterLookup.direct(lookupScope('by_project', '1'), ['u1'])]);
510
549
  return [{ id: 1 }, { id: 2 }];
511
550
  }),
512
551
  writeCheckpoint: null,
@@ -516,14 +555,14 @@ bucket_definitions:
516
555
  checkpoint: {
517
556
  buckets: [
518
557
  {
519
- bucket: 'by_project[1]',
558
+ bucket: '3#by_project[1]',
520
559
  checksum: 1,
521
560
  count: 1,
522
561
  priority: 3,
523
562
  subscriptions: [{ default: 0 }]
524
563
  },
525
564
  {
526
- bucket: 'by_project[2]',
565
+ bucket: '3#by_project[2]',
527
566
  checksum: 1,
528
567
  count: 1,
529
568
  priority: 3,
@@ -541,33 +580,33 @@ bucket_definitions:
541
580
  write_checkpoint: undefined
542
581
  }
543
582
  });
544
- expect(line.bucketsToFetch).toEqual([
583
+ expect(line.bucketsToFetch).toMatchObject([
545
584
  {
546
- bucket: 'by_project[1]',
585
+ bucket: '3#by_project[1]',
547
586
  priority: 3
548
587
  },
549
588
  {
550
- bucket: 'by_project[2]',
589
+ bucket: '3#by_project[2]',
551
590
  priority: 3
552
591
  }
553
592
  ]);
554
593
  line.advance();
555
594
  // This is the bucket data to be fetched
556
- expect(line.getFilteredBucketPositions()).toEqual(
595
+ expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(
557
596
  new Map([
558
- ['by_project[1]', 0n],
559
- ['by_project[2]', 0n]
597
+ ['3#by_project[1]', 0n],
598
+ ['3#by_project[2]', 0n]
560
599
  ])
561
600
  );
562
601
 
563
602
  line.advance();
564
- line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
565
- line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
603
+ line.updateBucketPosition({ bucket: '3#by_project[1]', nextAfter: 1n, hasMore: false });
604
+ line.updateBucketPosition({ bucket: '3#by_project[2]', nextAfter: 1n, hasMore: false });
566
605
 
567
606
  // Now we get a new line
568
607
  const line2 = (await state.buildNextCheckpointLine({
569
608
  base: storage.makeCheckpoint(2n, (lookups) => {
570
- expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]);
609
+ expect(lookups).toEqual([ScopedParameterLookup.direct(lookupScope('by_project', '1'), ['u1'])]);
571
610
  return [{ id: 1 }, { id: 2 }, { id: 3 }];
572
611
  }),
573
612
  writeCheckpoint: null,
@@ -584,7 +623,7 @@ bucket_definitions:
584
623
  removed_buckets: [],
585
624
  updated_buckets: [
586
625
  {
587
- bucket: 'by_project[3]',
626
+ bucket: '3#by_project[3]',
588
627
  checksum: 1,
589
628
  count: 1,
590
629
  priority: 3,
@@ -595,7 +634,7 @@ bucket_definitions:
595
634
  write_checkpoint: undefined
596
635
  }
597
636
  });
598
- expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
637
+ expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['3#by_project[3]', 0n]]));
599
638
  });
600
639
 
601
640
  describe('streams', () => {
@@ -820,6 +859,206 @@ config:
820
859
  });
821
860
  });
822
861
  });
862
+
863
+ describe('parameter query result logging', () => {
864
+ test('logs parameter query results for dynamic buckets', async () => {
865
+ const storage = new MockBucketChecksumStateStorage();
866
+ storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
867
+ storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
868
+ storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
869
+
870
+ const logMessages: string[] = [];
871
+ const logData: any[] = [];
872
+ const mockLogger = {
873
+ info: (message: string, data: any) => {
874
+ logMessages.push(message);
875
+ logData.push(data);
876
+ },
877
+ error: () => {},
878
+ warn: () => {},
879
+ debug: () => {}
880
+ };
881
+
882
+ const state = new BucketChecksumState({
883
+ syncContext,
884
+ tokenPayload: new JwtPayload({ sub: 'u1' }),
885
+ syncRequest,
886
+ syncRules: SYNC_RULES_DYNAMIC,
887
+ bucketStorage: storage,
888
+ logger: mockLogger as any
889
+ });
890
+
891
+ const line = (await state.buildNextCheckpointLine({
892
+ base: storage.makeCheckpoint(1n, (lookups) => {
893
+ return [{ id: 1 }, { id: 2 }, { id: 3 }];
894
+ }),
895
+ writeCheckpoint: null,
896
+ update: CHECKPOINT_INVALIDATE_ALL
897
+ }))!;
898
+ line.advance();
899
+
900
+ expect(logMessages[0]).toContain('param_results: 3');
901
+ expect(logData[0].parameter_query_results).toBe(3);
902
+ });
903
+
904
+ test('throws error with breakdown when parameter query limit is exceeded', async () => {
905
+ const SYNC_RULES_MULTI = SqlSyncRules.fromYaml(
906
+ `
907
+ bucket_definitions:
908
+ projects:
909
+ parameters: select id from projects where user_id = request.user_id()
910
+ data: []
911
+ tasks:
912
+ parameters: select id from tasks where user_id = request.user_id()
913
+ data: []
914
+ comments:
915
+ parameters: select id from comments where user_id = request.user_id()
916
+ data: []
917
+ `,
918
+ { defaultSchema: 'public' }
919
+ ).config.hydrate({ hydrationState: versionedHydrationState(4) });
920
+
921
+ const storage = new MockBucketChecksumStateStorage();
922
+
923
+ const errorMessages: string[] = [];
924
+ const errorData: any[] = [];
925
+ const mockLogger = {
926
+ info: () => {},
927
+ error: (message: string, data: any) => {
928
+ errorMessages.push(message);
929
+ errorData.push(data);
930
+ },
931
+ warn: () => {},
932
+ debug: () => {}
933
+ };
934
+
935
+ const smallContext = new SyncContext({
936
+ maxBuckets: 100,
937
+ maxParameterQueryResults: 50,
938
+ maxDataFetchConcurrency: 10
939
+ });
940
+
941
+ const state = new BucketChecksumState({
942
+ syncContext: smallContext,
943
+ tokenPayload: new JwtPayload({ sub: 'u1' }),
944
+ syncRequest,
945
+ syncRules: SYNC_RULES_MULTI,
946
+ bucketStorage: storage,
947
+ logger: mockLogger as any
948
+ });
949
+
950
+ // Create 60 total results: 30 projects + 20 tasks + 10 comments
951
+ const projectIds = Array.from({ length: 30 }, (_, i) => ({ id: i + 1 }));
952
+ const taskIds = Array.from({ length: 20 }, (_, i) => ({ id: i + 1 }));
953
+ const commentIds = Array.from({ length: 10 }, (_, i) => ({ id: i + 1 }));
954
+
955
+ for (let i = 1; i <= 30; i++) {
956
+ storage.updateTestChecksum({ bucket: `projects[${i}]`, checksum: 1, count: 1 });
957
+ }
958
+ for (let i = 1; i <= 20; i++) {
959
+ storage.updateTestChecksum({ bucket: `tasks[${i}]`, checksum: 1, count: 1 });
960
+ }
961
+ for (let i = 1; i <= 10; i++) {
962
+ storage.updateTestChecksum({ bucket: `comments[${i}]`, checksum: 1, count: 1 });
963
+ }
964
+
965
+ await expect(
966
+ state.buildNextCheckpointLine({
967
+ base: storage.makeCheckpoint(1n, (lookups) => {
968
+ const lookup = lookups[0];
969
+ const lookupName = lookup.values[0];
970
+ if (lookupName === 'projects') {
971
+ return projectIds;
972
+ } else if (lookupName === 'tasks') {
973
+ return taskIds;
974
+ } else {
975
+ return commentIds;
976
+ }
977
+ }),
978
+ writeCheckpoint: null,
979
+ update: CHECKPOINT_INVALIDATE_ALL
980
+ })
981
+ ).rejects.toThrow('Too many parameter query results: 60 (limit of 50)');
982
+
983
+ // Verify error log includes breakdown
984
+ expect(errorMessages[0]).toContain('Parameter query results by definition:');
985
+ expect(errorMessages[0]).toContain('projects: 30');
986
+ expect(errorMessages[0]).toContain('tasks: 20');
987
+ expect(errorMessages[0]).toContain('comments: 10');
988
+
989
+ expect(errorData[0].parameter_query_results).toBe(60);
990
+ expect(errorData[0].parameter_query_results_by_definition).toEqual({
991
+ projects: 30,
992
+ tasks: 20,
993
+ comments: 10
994
+ });
995
+ });
996
+
997
+ test('limits breakdown to top 10 definitions', async () => {
998
+ // Create sync rules with 15 different definitions with dynamic parameters
999
+ let yamlDefinitions = 'bucket_definitions:\n';
1000
+ for (let i = 1; i <= 15; i++) {
1001
+ yamlDefinitions += ` def${i}:\n`;
1002
+ yamlDefinitions += ` parameters: select id from def${i}_table where user_id = request.user_id()\n`;
1003
+ yamlDefinitions += ` data: []\n`;
1004
+ }
1005
+
1006
+ const SYNC_RULES_MANY = SqlSyncRules.fromYaml(yamlDefinitions, { defaultSchema: 'public' }).config.hydrate({
1007
+ hydrationState: versionedHydrationState(5)
1008
+ });
1009
+
1010
+ const storage = new MockBucketChecksumStateStorage();
1011
+
1012
+ const errorMessages: string[] = [];
1013
+ const mockLogger = {
1014
+ info: () => {},
1015
+ error: (message: string) => {
1016
+ errorMessages.push(message);
1017
+ },
1018
+ warn: () => {},
1019
+ debug: () => {}
1020
+ };
1021
+
1022
+ const smallContext = new SyncContext({
1023
+ maxBuckets: 100,
1024
+ maxParameterQueryResults: 10,
1025
+ maxDataFetchConcurrency: 10
1026
+ });
1027
+
1028
+ const state = new BucketChecksumState({
1029
+ syncContext: smallContext,
1030
+ tokenPayload: new JwtPayload({ sub: 'u1' }),
1031
+ syncRequest,
1032
+ syncRules: SYNC_RULES_MANY,
1033
+ bucketStorage: storage,
1034
+ logger: mockLogger as any
1035
+ });
1036
+
1037
+ // Each definition creates one bucket, total 15 buckets
1038
+ for (let i = 1; i <= 15; i++) {
1039
+ storage.updateTestChecksum({ bucket: `def${i}[${i}]`, checksum: 1, count: 1 });
1040
+ }
1041
+
1042
+ await expect(
1043
+ state.buildNextCheckpointLine({
1044
+ base: storage.makeCheckpoint(1n, (lookups) => {
1045
+ // Return one result for each definition
1046
+ return [{ id: 1 }];
1047
+ }),
1048
+ writeCheckpoint: null,
1049
+ update: CHECKPOINT_INVALIDATE_ALL
1050
+ })
1051
+ ).rejects.toThrow('Too many parameter query results: 15 (limit of 10)');
1052
+
1053
+ // Verify only top 10 are shown
1054
+ const errorMessage = errorMessages[0];
1055
+ expect(errorMessage).toContain('... and 5 more results from 5 definitions');
1056
+
1057
+ // Count how many definitions are listed (should be exactly 10)
1058
+ const defMatches = errorMessage.match(/def\d+:/g);
1059
+ expect(defMatches?.length).toBe(10);
1060
+ });
1061
+ });
823
1062
  });
824
1063
 
825
1064
  class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
@@ -837,9 +1076,9 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
837
1076
  this.filter?.({ invalidate: true });
838
1077
  }
839
1078
 
840
- async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
1079
+ async getChecksums(_checkpoint: InternalOpId, buckets: { bucket: string }[]): Promise<ChecksumMap> {
841
1080
  return new Map<string, BucketChecksum>(
842
- buckets.map((bucket) => {
1081
+ buckets.map(({ bucket }) => {
843
1082
  const checksum = this.state.get(bucket);
844
1083
  return [
845
1084
  bucket,