@powersync/service-core 1.19.2 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/dist/api/diagnostics.js +11 -4
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +13 -2
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/config-command.js +2 -2
- package/dist/entry/commands/config-command.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +84 -0
- package/dist/routes/endpoints/admin.d.ts +168 -0
- package/dist/routes/endpoints/admin.js +33 -20
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +6 -9
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +43 -15
- package/dist/storage/BucketStorageFactory.js +70 -1
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +28 -2
- package/dist/storage/PersistedSyncRulesContent.js +79 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +20 -0
- package/dist/storage/StorageVersionConfig.js +20 -0
- package/dist/storage/StorageVersionConfig.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +2 -1
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +6 -2
- package/dist/sync/BucketChecksumState.js +85 -10
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/util/config/collectors/config-collector.js +13 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +4 -4
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +2 -2
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +3 -3
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -1
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/env.d.ts +1 -0
- package/dist/util/env.js +5 -0
- package/dist/util/env.js.map +1 -1
- package/package.json +6 -6
- package/src/api/diagnostics.ts +12 -4
- package/src/entry/commands/compact-action.ts +15 -2
- package/src/entry/commands/config-command.ts +3 -3
- package/src/replication/AbstractReplicator.ts +3 -5
- package/src/routes/endpoints/admin.ts +42 -25
- package/src/routes/endpoints/sync-rules.ts +14 -13
- package/src/storage/BucketStorageFactory.ts +110 -19
- package/src/storage/PersistedSyncRulesContent.ts +114 -4
- package/src/storage/StorageVersionConfig.ts +30 -0
- package/src/storage/SyncRulesBucketStorage.ts +2 -1
- package/src/storage/storage-index.ts +1 -0
- package/src/sync/BucketChecksumState.ts +129 -16
- package/src/util/config/collectors/config-collector.ts +16 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +5 -5
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +4 -4
- package/src/util/config/types.ts +1 -2
- package/src/util/env.ts +5 -0
- package/test/src/config.test.ts +115 -0
- package/test/src/routes/admin.test.ts +48 -0
- package/test/src/routes/mocks.ts +22 -1
- package/test/src/routes/stream.test.ts +3 -2
- package/test/src/sync/BucketChecksumState.test.ts +285 -78
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -13,8 +13,13 @@ import {
|
|
|
13
13
|
WatchFilterEvent
|
|
14
14
|
} from '@/index.js';
|
|
15
15
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
16
|
-
import {
|
|
17
|
-
|
|
16
|
+
import {
|
|
17
|
+
RequestJwtPayload,
|
|
18
|
+
ScopedParameterLookup,
|
|
19
|
+
SqliteJsonRow,
|
|
20
|
+
SqlSyncRules,
|
|
21
|
+
versionedHydrationState
|
|
22
|
+
} from '@powersync/service-sync-rules';
|
|
18
23
|
import { beforeEach, describe, expect, test } from 'vitest';
|
|
19
24
|
|
|
20
25
|
describe('BucketChecksumState', () => {
|
|
@@ -65,7 +70,7 @@ bucket_definitions:
|
|
|
65
70
|
test('global bucket with update', async () => {
|
|
66
71
|
const storage = new MockBucketChecksumStateStorage();
|
|
67
72
|
// Set intial state
|
|
68
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
73
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
69
74
|
|
|
70
75
|
const state = new BucketChecksumState({
|
|
71
76
|
syncContext,
|
|
@@ -83,7 +88,7 @@ bucket_definitions:
|
|
|
83
88
|
line.advance();
|
|
84
89
|
expect(line.checkpointLine).toEqual({
|
|
85
90
|
checkpoint: {
|
|
86
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
91
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
87
92
|
last_op_id: '1',
|
|
88
93
|
write_checkpoint: undefined,
|
|
89
94
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
@@ -91,26 +96,26 @@ bucket_definitions:
|
|
|
91
96
|
});
|
|
92
97
|
expect(line.bucketsToFetch).toEqual([
|
|
93
98
|
{
|
|
94
|
-
bucket: 'global[]',
|
|
99
|
+
bucket: '1#global[]',
|
|
95
100
|
priority: 3
|
|
96
101
|
}
|
|
97
102
|
]);
|
|
98
103
|
// This is the bucket data to be fetched
|
|
99
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
|
|
104
|
+
expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
|
|
100
105
|
|
|
101
106
|
// This similuates the bucket data being sent
|
|
102
107
|
line.advance();
|
|
103
|
-
line.updateBucketPosition({ bucket: 'global[]', nextAfter: 1n, hasMore: false });
|
|
108
|
+
line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: false });
|
|
104
109
|
|
|
105
110
|
// Update bucket storage state
|
|
106
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 2, count: 2 });
|
|
111
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
|
|
107
112
|
|
|
108
113
|
// Now we get a new line
|
|
109
114
|
const line2 = (await state.buildNextCheckpointLine({
|
|
110
115
|
base: storage.makeCheckpoint(2n),
|
|
111
116
|
writeCheckpoint: null,
|
|
112
117
|
update: {
|
|
113
|
-
updatedDataBuckets: new Set(['global[]']),
|
|
118
|
+
updatedDataBuckets: new Set(['1#global[]']),
|
|
114
119
|
invalidateDataBuckets: false,
|
|
115
120
|
updatedParameterLookups: new Set(),
|
|
116
121
|
invalidateParameterBuckets: false
|
|
@@ -120,12 +125,14 @@ bucket_definitions:
|
|
|
120
125
|
expect(line2.checkpointLine).toEqual({
|
|
121
126
|
checkpoint_diff: {
|
|
122
127
|
removed_buckets: [],
|
|
123
|
-
updated_buckets: [
|
|
128
|
+
updated_buckets: [
|
|
129
|
+
{ bucket: '1#global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
130
|
+
],
|
|
124
131
|
last_op_id: '2',
|
|
125
132
|
write_checkpoint: undefined
|
|
126
133
|
}
|
|
127
134
|
});
|
|
128
|
-
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
|
|
135
|
+
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
|
|
129
136
|
});
|
|
130
137
|
|
|
131
138
|
test('global bucket with initial state', async () => {
|
|
@@ -134,13 +141,13 @@ bucket_definitions:
|
|
|
134
141
|
/// (getFilteredBucketStates)
|
|
135
142
|
const storage = new MockBucketChecksumStateStorage();
|
|
136
143
|
// Set intial state
|
|
137
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
144
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
138
145
|
|
|
139
146
|
const state = new BucketChecksumState({
|
|
140
147
|
syncContext,
|
|
141
148
|
tokenPayload,
|
|
142
149
|
// Client sets the initial state here
|
|
143
|
-
syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
|
|
150
|
+
syncRequest: { buckets: [{ name: '1#global[]', after: '1' }] },
|
|
144
151
|
syncRules: SYNC_RULES_GLOBAL,
|
|
145
152
|
bucketStorage: storage
|
|
146
153
|
});
|
|
@@ -153,7 +160,7 @@ bucket_definitions:
|
|
|
153
160
|
line.advance();
|
|
154
161
|
expect(line.checkpointLine).toEqual({
|
|
155
162
|
checkpoint: {
|
|
156
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
163
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
157
164
|
last_op_id: '1',
|
|
158
165
|
write_checkpoint: undefined,
|
|
159
166
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
@@ -161,19 +168,19 @@ bucket_definitions:
|
|
|
161
168
|
});
|
|
162
169
|
expect(line.bucketsToFetch).toEqual([
|
|
163
170
|
{
|
|
164
|
-
bucket: 'global[]',
|
|
171
|
+
bucket: '1#global[]',
|
|
165
172
|
priority: 3
|
|
166
173
|
}
|
|
167
174
|
]);
|
|
168
175
|
// This is the main difference between this and the previous test
|
|
169
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
|
|
176
|
+
expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
|
|
170
177
|
});
|
|
171
178
|
|
|
172
179
|
test('multiple static buckets', async () => {
|
|
173
180
|
const storage = new MockBucketChecksumStateStorage();
|
|
174
181
|
// Set intial state
|
|
175
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
176
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
182
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
183
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
177
184
|
|
|
178
185
|
const state = new BucketChecksumState({
|
|
179
186
|
syncContext,
|
|
@@ -191,8 +198,8 @@ bucket_definitions:
|
|
|
191
198
|
expect(line.checkpointLine).toEqual({
|
|
192
199
|
checkpoint: {
|
|
193
200
|
buckets: [
|
|
194
|
-
{ bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
195
|
-
{ bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
201
|
+
{ bucket: '2#global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
202
|
+
{ bucket: '2#global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
196
203
|
],
|
|
197
204
|
last_op_id: '1',
|
|
198
205
|
write_checkpoint: undefined,
|
|
@@ -201,25 +208,25 @@ bucket_definitions:
|
|
|
201
208
|
});
|
|
202
209
|
expect(line.bucketsToFetch).toEqual([
|
|
203
210
|
{
|
|
204
|
-
bucket: 'global[1]',
|
|
211
|
+
bucket: '2#global[1]',
|
|
205
212
|
priority: 3
|
|
206
213
|
},
|
|
207
214
|
{
|
|
208
|
-
bucket: 'global[2]',
|
|
215
|
+
bucket: '2#global[2]',
|
|
209
216
|
priority: 3
|
|
210
217
|
}
|
|
211
218
|
]);
|
|
212
219
|
line.advance();
|
|
213
220
|
|
|
214
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
215
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
221
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
222
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
216
223
|
|
|
217
224
|
const line2 = (await state.buildNextCheckpointLine({
|
|
218
225
|
base: storage.makeCheckpoint(2n),
|
|
219
226
|
writeCheckpoint: null,
|
|
220
227
|
update: {
|
|
221
228
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
222
|
-
updatedDataBuckets: new Set(['global[1]', 'global[2]']),
|
|
229
|
+
updatedDataBuckets: new Set(['2#global[1]', '2#global[2]']),
|
|
223
230
|
invalidateDataBuckets: false
|
|
224
231
|
}
|
|
225
232
|
}))!;
|
|
@@ -227,8 +234,8 @@ bucket_definitions:
|
|
|
227
234
|
checkpoint_diff: {
|
|
228
235
|
removed_buckets: [],
|
|
229
236
|
updated_buckets: [
|
|
230
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
231
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
237
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
238
|
+
{ bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
232
239
|
],
|
|
233
240
|
last_op_id: '2',
|
|
234
241
|
write_checkpoint: undefined
|
|
@@ -251,7 +258,7 @@ bucket_definitions:
|
|
|
251
258
|
bucketStorage: storage
|
|
252
259
|
});
|
|
253
260
|
|
|
254
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
261
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
255
262
|
|
|
256
263
|
const line = (await state.buildNextCheckpointLine({
|
|
257
264
|
base: storage.makeCheckpoint(1n),
|
|
@@ -261,7 +268,7 @@ bucket_definitions:
|
|
|
261
268
|
line.advance();
|
|
262
269
|
expect(line.checkpointLine).toEqual({
|
|
263
270
|
checkpoint: {
|
|
264
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
271
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
265
272
|
last_op_id: '1',
|
|
266
273
|
write_checkpoint: undefined,
|
|
267
274
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
@@ -269,11 +276,11 @@ bucket_definitions:
|
|
|
269
276
|
});
|
|
270
277
|
expect(line.bucketsToFetch).toEqual([
|
|
271
278
|
{
|
|
272
|
-
bucket: 'global[]',
|
|
279
|
+
bucket: '1#global[]',
|
|
273
280
|
priority: 3
|
|
274
281
|
}
|
|
275
282
|
]);
|
|
276
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
|
|
283
|
+
expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
|
|
277
284
|
});
|
|
278
285
|
|
|
279
286
|
test('invalidating individual bucket', async () => {
|
|
@@ -281,8 +288,8 @@ bucket_definitions:
|
|
|
281
288
|
|
|
282
289
|
const storage = new MockBucketChecksumStateStorage();
|
|
283
290
|
// Set initial state
|
|
284
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
285
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
291
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
292
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
286
293
|
|
|
287
294
|
const state = new BucketChecksumState({
|
|
288
295
|
syncContext,
|
|
@@ -301,11 +308,11 @@ bucket_definitions:
|
|
|
301
308
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
302
309
|
});
|
|
303
310
|
line!.advance();
|
|
304
|
-
line!.updateBucketPosition({ bucket: 'global[1]', nextAfter: 1n, hasMore: false });
|
|
305
|
-
line!.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: false });
|
|
311
|
+
line!.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 1n, hasMore: false });
|
|
312
|
+
line!.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: false });
|
|
306
313
|
|
|
307
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
308
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
314
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
315
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
309
316
|
|
|
310
317
|
const line2 = (await state.buildNextCheckpointLine({
|
|
311
318
|
base: storage.makeCheckpoint(2n),
|
|
@@ -315,7 +322,7 @@ bucket_definitions:
|
|
|
315
322
|
// Invalidate the state for global[1] - will only re-check the single bucket.
|
|
316
323
|
// This is essentially inconsistent state, but is the simplest way to test that
|
|
317
324
|
// the filter is working.
|
|
318
|
-
updatedDataBuckets: new Set(['global[1]']),
|
|
325
|
+
updatedDataBuckets: new Set(['2#global[1]']),
|
|
319
326
|
invalidateDataBuckets: false
|
|
320
327
|
}
|
|
321
328
|
}))!;
|
|
@@ -324,13 +331,13 @@ bucket_definitions:
|
|
|
324
331
|
removed_buckets: [],
|
|
325
332
|
updated_buckets: [
|
|
326
333
|
// This does not include global[2], since it was not invalidated.
|
|
327
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
334
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
328
335
|
],
|
|
329
336
|
last_op_id: '2',
|
|
330
337
|
write_checkpoint: undefined
|
|
331
338
|
}
|
|
332
339
|
});
|
|
333
|
-
expect(line2.bucketsToFetch).toEqual([{ bucket: 'global[1]', priority: 3 }]);
|
|
340
|
+
expect(line2.bucketsToFetch).toEqual([{ bucket: '2#global[1]', priority: 3 }]);
|
|
334
341
|
});
|
|
335
342
|
|
|
336
343
|
test('invalidating all buckets', async () => {
|
|
@@ -349,8 +356,8 @@ bucket_definitions:
|
|
|
349
356
|
// storage.filter = state.checkpointFilter;
|
|
350
357
|
|
|
351
358
|
// Set initial state
|
|
352
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
353
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
359
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
360
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
354
361
|
|
|
355
362
|
const line = await state.buildNextCheckpointLine({
|
|
356
363
|
base: storage.makeCheckpoint(1n),
|
|
@@ -360,8 +367,8 @@ bucket_definitions:
|
|
|
360
367
|
|
|
361
368
|
line!.advance();
|
|
362
369
|
|
|
363
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
364
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
370
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
371
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
365
372
|
|
|
366
373
|
const line2 = (await state.buildNextCheckpointLine({
|
|
367
374
|
base: storage.makeCheckpoint(2n),
|
|
@@ -373,24 +380,24 @@ bucket_definitions:
|
|
|
373
380
|
checkpoint_diff: {
|
|
374
381
|
removed_buckets: [],
|
|
375
382
|
updated_buckets: [
|
|
376
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
377
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
383
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
384
|
+
{ bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
378
385
|
],
|
|
379
386
|
last_op_id: '2',
|
|
380
387
|
write_checkpoint: undefined
|
|
381
388
|
}
|
|
382
389
|
});
|
|
383
390
|
expect(line2.bucketsToFetch).toEqual([
|
|
384
|
-
{ bucket: 'global[1]', priority: 3 },
|
|
385
|
-
{ bucket: 'global[2]', priority: 3 }
|
|
391
|
+
{ bucket: '2#global[1]', priority: 3 },
|
|
392
|
+
{ bucket: '2#global[2]', priority: 3 }
|
|
386
393
|
]);
|
|
387
394
|
});
|
|
388
395
|
|
|
389
396
|
test('interrupt and resume static buckets checkpoint', async () => {
|
|
390
397
|
const storage = new MockBucketChecksumStateStorage();
|
|
391
398
|
// Set intial state
|
|
392
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 3, count: 3 });
|
|
393
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 3, count: 3 });
|
|
399
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 3, count: 3 });
|
|
400
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 3, count: 3 });
|
|
394
401
|
|
|
395
402
|
const state = new BucketChecksumState({
|
|
396
403
|
syncContext,
|
|
@@ -409,8 +416,8 @@ bucket_definitions:
|
|
|
409
416
|
expect(line.checkpointLine).toEqual({
|
|
410
417
|
checkpoint: {
|
|
411
418
|
buckets: [
|
|
412
|
-
{ bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
413
|
-
{ bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
419
|
+
{ bucket: '2#global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
420
|
+
{ bucket: '2#global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
414
421
|
],
|
|
415
422
|
last_op_id: '3',
|
|
416
423
|
write_checkpoint: undefined,
|
|
@@ -419,11 +426,11 @@ bucket_definitions:
|
|
|
419
426
|
});
|
|
420
427
|
expect(line.bucketsToFetch).toEqual([
|
|
421
428
|
{
|
|
422
|
-
bucket: 'global[1]',
|
|
429
|
+
bucket: '2#global[1]',
|
|
423
430
|
priority: 3
|
|
424
431
|
},
|
|
425
432
|
{
|
|
426
|
-
bucket: 'global[2]',
|
|
433
|
+
bucket: '2#global[2]',
|
|
427
434
|
priority: 3
|
|
428
435
|
}
|
|
429
436
|
]);
|
|
@@ -431,17 +438,17 @@ bucket_definitions:
|
|
|
431
438
|
// This is the bucket data to be fetched
|
|
432
439
|
expect(line.getFilteredBucketPositions()).toEqual(
|
|
433
440
|
new Map([
|
|
434
|
-
['global[1]', 0n],
|
|
435
|
-
['global[2]', 0n]
|
|
441
|
+
['2#global[1]', 0n],
|
|
442
|
+
['2#global[2]', 0n]
|
|
436
443
|
])
|
|
437
444
|
);
|
|
438
445
|
|
|
439
446
|
// No data changes here.
|
|
440
447
|
// We simulate partial data sent, before a checkpoint is interrupted.
|
|
441
448
|
line.advance();
|
|
442
|
-
line.updateBucketPosition({ bucket: 'global[1]', nextAfter: 3n, hasMore: false });
|
|
443
|
-
line.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: true });
|
|
444
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
|
|
449
|
+
line.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 3n, hasMore: false });
|
|
450
|
+
line.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: true });
|
|
451
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 4, count: 4 });
|
|
445
452
|
|
|
446
453
|
const line2 = (await state.buildNextCheckpointLine({
|
|
447
454
|
base: storage.makeCheckpoint(4n),
|
|
@@ -449,7 +456,7 @@ bucket_definitions:
|
|
|
449
456
|
update: {
|
|
450
457
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
451
458
|
invalidateDataBuckets: false,
|
|
452
|
-
updatedDataBuckets: new Set(['global[1]'])
|
|
459
|
+
updatedDataBuckets: new Set(['2#global[1]'])
|
|
453
460
|
}
|
|
454
461
|
}))!;
|
|
455
462
|
line2.advance();
|
|
@@ -458,7 +465,7 @@ bucket_definitions:
|
|
|
458
465
|
removed_buckets: [],
|
|
459
466
|
updated_buckets: [
|
|
460
467
|
{
|
|
461
|
-
bucket: 'global[1]',
|
|
468
|
+
bucket: '2#global[1]',
|
|
462
469
|
checksum: 4,
|
|
463
470
|
count: 4,
|
|
464
471
|
priority: 3,
|
|
@@ -472,19 +479,19 @@ bucket_definitions:
|
|
|
472
479
|
// This should contain both buckets, even though only one changed.
|
|
473
480
|
expect(line2.bucketsToFetch).toEqual([
|
|
474
481
|
{
|
|
475
|
-
bucket: 'global[1]',
|
|
482
|
+
bucket: '2#global[1]',
|
|
476
483
|
priority: 3
|
|
477
484
|
},
|
|
478
485
|
{
|
|
479
|
-
bucket: 'global[2]',
|
|
486
|
+
bucket: '2#global[2]',
|
|
480
487
|
priority: 3
|
|
481
488
|
}
|
|
482
489
|
]);
|
|
483
490
|
|
|
484
491
|
expect(line2.getFilteredBucketPositions()).toEqual(
|
|
485
492
|
new Map([
|
|
486
|
-
['global[1]', 3n],
|
|
487
|
-
['global[2]', 1n]
|
|
493
|
+
['2#global[1]', 3n],
|
|
494
|
+
['2#global[2]', 1n]
|
|
488
495
|
])
|
|
489
496
|
);
|
|
490
497
|
});
|
|
@@ -492,9 +499,9 @@ bucket_definitions:
|
|
|
492
499
|
test('dynamic buckets with updates', async () => {
|
|
493
500
|
const storage = new MockBucketChecksumStateStorage();
|
|
494
501
|
// Set intial state
|
|
495
|
-
storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
|
|
496
|
-
storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
|
|
497
|
-
storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
|
|
502
|
+
storage.updateTestChecksum({ bucket: '3#by_project[1]', checksum: 1, count: 1 });
|
|
503
|
+
storage.updateTestChecksum({ bucket: '3#by_project[2]', checksum: 1, count: 1 });
|
|
504
|
+
storage.updateTestChecksum({ bucket: '3#by_project[3]', checksum: 1, count: 1 });
|
|
498
505
|
|
|
499
506
|
const state = new BucketChecksumState({
|
|
500
507
|
syncContext,
|
|
@@ -516,14 +523,14 @@ bucket_definitions:
|
|
|
516
523
|
checkpoint: {
|
|
517
524
|
buckets: [
|
|
518
525
|
{
|
|
519
|
-
bucket: 'by_project[1]',
|
|
526
|
+
bucket: '3#by_project[1]',
|
|
520
527
|
checksum: 1,
|
|
521
528
|
count: 1,
|
|
522
529
|
priority: 3,
|
|
523
530
|
subscriptions: [{ default: 0 }]
|
|
524
531
|
},
|
|
525
532
|
{
|
|
526
|
-
bucket: 'by_project[2]',
|
|
533
|
+
bucket: '3#by_project[2]',
|
|
527
534
|
checksum: 1,
|
|
528
535
|
count: 1,
|
|
529
536
|
priority: 3,
|
|
@@ -543,11 +550,11 @@ bucket_definitions:
|
|
|
543
550
|
});
|
|
544
551
|
expect(line.bucketsToFetch).toEqual([
|
|
545
552
|
{
|
|
546
|
-
bucket: 'by_project[1]',
|
|
553
|
+
bucket: '3#by_project[1]',
|
|
547
554
|
priority: 3
|
|
548
555
|
},
|
|
549
556
|
{
|
|
550
|
-
bucket: 'by_project[2]',
|
|
557
|
+
bucket: '3#by_project[2]',
|
|
551
558
|
priority: 3
|
|
552
559
|
}
|
|
553
560
|
]);
|
|
@@ -555,14 +562,14 @@ bucket_definitions:
|
|
|
555
562
|
// This is the bucket data to be fetched
|
|
556
563
|
expect(line.getFilteredBucketPositions()).toEqual(
|
|
557
564
|
new Map([
|
|
558
|
-
['by_project[1]', 0n],
|
|
559
|
-
['by_project[2]', 0n]
|
|
565
|
+
['3#by_project[1]', 0n],
|
|
566
|
+
['3#by_project[2]', 0n]
|
|
560
567
|
])
|
|
561
568
|
);
|
|
562
569
|
|
|
563
570
|
line.advance();
|
|
564
|
-
line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
|
|
565
|
-
line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
|
|
571
|
+
line.updateBucketPosition({ bucket: '3#by_project[1]', nextAfter: 1n, hasMore: false });
|
|
572
|
+
line.updateBucketPosition({ bucket: '3#by_project[2]', nextAfter: 1n, hasMore: false });
|
|
566
573
|
|
|
567
574
|
// Now we get a new line
|
|
568
575
|
const line2 = (await state.buildNextCheckpointLine({
|
|
@@ -584,7 +591,7 @@ bucket_definitions:
|
|
|
584
591
|
removed_buckets: [],
|
|
585
592
|
updated_buckets: [
|
|
586
593
|
{
|
|
587
|
-
bucket: 'by_project[3]',
|
|
594
|
+
bucket: '3#by_project[3]',
|
|
588
595
|
checksum: 1,
|
|
589
596
|
count: 1,
|
|
590
597
|
priority: 3,
|
|
@@ -595,7 +602,7 @@ bucket_definitions:
|
|
|
595
602
|
write_checkpoint: undefined
|
|
596
603
|
}
|
|
597
604
|
});
|
|
598
|
-
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
|
|
605
|
+
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['3#by_project[3]', 0n]]));
|
|
599
606
|
});
|
|
600
607
|
|
|
601
608
|
describe('streams', () => {
|
|
@@ -820,6 +827,206 @@ config:
|
|
|
820
827
|
});
|
|
821
828
|
});
|
|
822
829
|
});
|
|
830
|
+
|
|
831
|
+
describe('parameter query result logging', () => {
|
|
832
|
+
test('logs parameter query results for dynamic buckets', async () => {
|
|
833
|
+
const storage = new MockBucketChecksumStateStorage();
|
|
834
|
+
storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
|
|
835
|
+
storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
|
|
836
|
+
storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
|
|
837
|
+
|
|
838
|
+
const logMessages: string[] = [];
|
|
839
|
+
const logData: any[] = [];
|
|
840
|
+
const mockLogger = {
|
|
841
|
+
info: (message: string, data: any) => {
|
|
842
|
+
logMessages.push(message);
|
|
843
|
+
logData.push(data);
|
|
844
|
+
},
|
|
845
|
+
error: () => {},
|
|
846
|
+
warn: () => {},
|
|
847
|
+
debug: () => {}
|
|
848
|
+
};
|
|
849
|
+
|
|
850
|
+
const state = new BucketChecksumState({
|
|
851
|
+
syncContext,
|
|
852
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
853
|
+
syncRequest,
|
|
854
|
+
syncRules: SYNC_RULES_DYNAMIC,
|
|
855
|
+
bucketStorage: storage,
|
|
856
|
+
logger: mockLogger as any
|
|
857
|
+
});
|
|
858
|
+
|
|
859
|
+
const line = (await state.buildNextCheckpointLine({
|
|
860
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
861
|
+
return [{ id: 1 }, { id: 2 }, { id: 3 }];
|
|
862
|
+
}),
|
|
863
|
+
writeCheckpoint: null,
|
|
864
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
865
|
+
}))!;
|
|
866
|
+
line.advance();
|
|
867
|
+
|
|
868
|
+
expect(logMessages[0]).toContain('param_results: 3');
|
|
869
|
+
expect(logData[0].parameter_query_results).toBe(3);
|
|
870
|
+
});
|
|
871
|
+
|
|
872
|
+
test('throws error with breakdown when parameter query limit is exceeded', async () => {
|
|
873
|
+
const SYNC_RULES_MULTI = SqlSyncRules.fromYaml(
|
|
874
|
+
`
|
|
875
|
+
bucket_definitions:
|
|
876
|
+
projects:
|
|
877
|
+
parameters: select id from projects where user_id = request.user_id()
|
|
878
|
+
data: []
|
|
879
|
+
tasks:
|
|
880
|
+
parameters: select id from tasks where user_id = request.user_id()
|
|
881
|
+
data: []
|
|
882
|
+
comments:
|
|
883
|
+
parameters: select id from comments where user_id = request.user_id()
|
|
884
|
+
data: []
|
|
885
|
+
`,
|
|
886
|
+
{ defaultSchema: 'public' }
|
|
887
|
+
).config.hydrate({ hydrationState: versionedHydrationState(4) });
|
|
888
|
+
|
|
889
|
+
const storage = new MockBucketChecksumStateStorage();
|
|
890
|
+
|
|
891
|
+
const errorMessages: string[] = [];
|
|
892
|
+
const errorData: any[] = [];
|
|
893
|
+
const mockLogger = {
|
|
894
|
+
info: () => {},
|
|
895
|
+
error: (message: string, data: any) => {
|
|
896
|
+
errorMessages.push(message);
|
|
897
|
+
errorData.push(data);
|
|
898
|
+
},
|
|
899
|
+
warn: () => {},
|
|
900
|
+
debug: () => {}
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
const smallContext = new SyncContext({
|
|
904
|
+
maxBuckets: 100,
|
|
905
|
+
maxParameterQueryResults: 50,
|
|
906
|
+
maxDataFetchConcurrency: 10
|
|
907
|
+
});
|
|
908
|
+
|
|
909
|
+
const state = new BucketChecksumState({
|
|
910
|
+
syncContext: smallContext,
|
|
911
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
912
|
+
syncRequest,
|
|
913
|
+
syncRules: SYNC_RULES_MULTI,
|
|
914
|
+
bucketStorage: storage,
|
|
915
|
+
logger: mockLogger as any
|
|
916
|
+
});
|
|
917
|
+
|
|
918
|
+
// Create 60 total results: 30 projects + 20 tasks + 10 comments
|
|
919
|
+
const projectIds = Array.from({ length: 30 }, (_, i) => ({ id: i + 1 }));
|
|
920
|
+
const taskIds = Array.from({ length: 20 }, (_, i) => ({ id: i + 1 }));
|
|
921
|
+
const commentIds = Array.from({ length: 10 }, (_, i) => ({ id: i + 1 }));
|
|
922
|
+
|
|
923
|
+
for (let i = 1; i <= 30; i++) {
|
|
924
|
+
storage.updateTestChecksum({ bucket: `projects[${i}]`, checksum: 1, count: 1 });
|
|
925
|
+
}
|
|
926
|
+
for (let i = 1; i <= 20; i++) {
|
|
927
|
+
storage.updateTestChecksum({ bucket: `tasks[${i}]`, checksum: 1, count: 1 });
|
|
928
|
+
}
|
|
929
|
+
for (let i = 1; i <= 10; i++) {
|
|
930
|
+
storage.updateTestChecksum({ bucket: `comments[${i}]`, checksum: 1, count: 1 });
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
await expect(
|
|
934
|
+
state.buildNextCheckpointLine({
|
|
935
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
936
|
+
const lookup = lookups[0];
|
|
937
|
+
const lookupName = lookup.values[0];
|
|
938
|
+
if (lookupName === 'projects') {
|
|
939
|
+
return projectIds;
|
|
940
|
+
} else if (lookupName === 'tasks') {
|
|
941
|
+
return taskIds;
|
|
942
|
+
} else {
|
|
943
|
+
return commentIds;
|
|
944
|
+
}
|
|
945
|
+
}),
|
|
946
|
+
writeCheckpoint: null,
|
|
947
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
948
|
+
})
|
|
949
|
+
).rejects.toThrow('Too many parameter query results: 60 (limit of 50)');
|
|
950
|
+
|
|
951
|
+
// Verify error log includes breakdown
|
|
952
|
+
expect(errorMessages[0]).toContain('Parameter query results by definition:');
|
|
953
|
+
expect(errorMessages[0]).toContain('projects: 30');
|
|
954
|
+
expect(errorMessages[0]).toContain('tasks: 20');
|
|
955
|
+
expect(errorMessages[0]).toContain('comments: 10');
|
|
956
|
+
|
|
957
|
+
expect(errorData[0].parameter_query_results).toBe(60);
|
|
958
|
+
expect(errorData[0].parameter_query_results_by_definition).toEqual({
|
|
959
|
+
projects: 30,
|
|
960
|
+
tasks: 20,
|
|
961
|
+
comments: 10
|
|
962
|
+
});
|
|
963
|
+
});
|
|
964
|
+
|
|
965
|
+
test('limits breakdown to top 10 definitions', async () => {
|
|
966
|
+
// Create sync rules with 15 different definitions with dynamic parameters
|
|
967
|
+
let yamlDefinitions = 'bucket_definitions:\n';
|
|
968
|
+
for (let i = 1; i <= 15; i++) {
|
|
969
|
+
yamlDefinitions += ` def${i}:\n`;
|
|
970
|
+
yamlDefinitions += ` parameters: select id from def${i}_table where user_id = request.user_id()\n`;
|
|
971
|
+
yamlDefinitions += ` data: []\n`;
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
const SYNC_RULES_MANY = SqlSyncRules.fromYaml(yamlDefinitions, { defaultSchema: 'public' }).config.hydrate({
|
|
975
|
+
hydrationState: versionedHydrationState(5)
|
|
976
|
+
});
|
|
977
|
+
|
|
978
|
+
const storage = new MockBucketChecksumStateStorage();
|
|
979
|
+
|
|
980
|
+
const errorMessages: string[] = [];
|
|
981
|
+
const mockLogger = {
|
|
982
|
+
info: () => {},
|
|
983
|
+
error: (message: string) => {
|
|
984
|
+
errorMessages.push(message);
|
|
985
|
+
},
|
|
986
|
+
warn: () => {},
|
|
987
|
+
debug: () => {}
|
|
988
|
+
};
|
|
989
|
+
|
|
990
|
+
const smallContext = new SyncContext({
|
|
991
|
+
maxBuckets: 100,
|
|
992
|
+
maxParameterQueryResults: 10,
|
|
993
|
+
maxDataFetchConcurrency: 10
|
|
994
|
+
});
|
|
995
|
+
|
|
996
|
+
const state = new BucketChecksumState({
|
|
997
|
+
syncContext: smallContext,
|
|
998
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
999
|
+
syncRequest,
|
|
1000
|
+
syncRules: SYNC_RULES_MANY,
|
|
1001
|
+
bucketStorage: storage,
|
|
1002
|
+
logger: mockLogger as any
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
// Each definition creates one bucket, total 15 buckets
|
|
1006
|
+
for (let i = 1; i <= 15; i++) {
|
|
1007
|
+
storage.updateTestChecksum({ bucket: `def${i}[${i}]`, checksum: 1, count: 1 });
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
await expect(
|
|
1011
|
+
state.buildNextCheckpointLine({
|
|
1012
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
1013
|
+
// Return one result for each definition
|
|
1014
|
+
return [{ id: 1 }];
|
|
1015
|
+
}),
|
|
1016
|
+
writeCheckpoint: null,
|
|
1017
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
1018
|
+
})
|
|
1019
|
+
).rejects.toThrow('Too many parameter query results: 15 (limit of 10)');
|
|
1020
|
+
|
|
1021
|
+
// Verify only top 10 are shown
|
|
1022
|
+
const errorMessage = errorMessages[0];
|
|
1023
|
+
expect(errorMessage).toContain('... and 5 more results from 5 definitions');
|
|
1024
|
+
|
|
1025
|
+
// Count how many definitions are listed (should be exactly 10)
|
|
1026
|
+
const defMatches = errorMessage.match(/def\d+:/g);
|
|
1027
|
+
expect(defMatches?.length).toBe(10);
|
|
1028
|
+
});
|
|
1029
|
+
});
|
|
823
1030
|
});
|
|
824
1031
|
|
|
825
1032
|
class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
|