@powersync/service-core 1.19.2 → 1.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/dist/api/diagnostics.js +11 -4
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +13 -2
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/config-command.js +2 -2
- package/dist/entry/commands/config-command.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +84 -0
- package/dist/routes/endpoints/admin.d.ts +168 -0
- package/dist/routes/endpoints/admin.js +34 -20
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +6 -9
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +6 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +21 -8
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +48 -15
- package/dist/storage/BucketStorageFactory.js +70 -1
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +5 -2
- package/dist/storage/ChecksumCache.js +8 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +33 -3
- package/dist/storage/PersistedSyncRulesContent.js +80 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/SourceTable.d.ts +7 -2
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +53 -0
- package/dist/storage/StorageVersionConfig.js +53 -0
- package/dist/storage/StorageVersionConfig.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +14 -4
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +8 -4
- package/dist/sync/BucketChecksumState.js +97 -52
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -0
- package/dist/sync/util.js +10 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/collectors/config-collector.js +13 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +4 -4
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +2 -2
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +3 -3
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -1
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/env.d.ts +1 -0
- package/dist/util/env.js +5 -0
- package/dist/util/env.js.map +1 -1
- package/package.json +6 -6
- package/src/api/diagnostics.ts +12 -4
- package/src/entry/commands/compact-action.ts +15 -2
- package/src/entry/commands/config-command.ts +3 -3
- package/src/replication/AbstractReplicator.ts +3 -5
- package/src/routes/endpoints/admin.ts +43 -25
- package/src/routes/endpoints/sync-rules.ts +14 -13
- package/src/routes/endpoints/sync-stream.ts +6 -1
- package/src/storage/BucketStorageBatch.ts +23 -9
- package/src/storage/BucketStorageFactory.ts +116 -19
- package/src/storage/ChecksumCache.ts +14 -6
- package/src/storage/PersistedSyncRulesContent.ts +119 -4
- package/src/storage/SourceTable.ts +7 -1
- package/src/storage/StorageVersionConfig.ts +78 -0
- package/src/storage/SyncRulesBucketStorage.ts +20 -4
- package/src/storage/storage-index.ts +1 -0
- package/src/sync/BucketChecksumState.ts +147 -65
- package/src/sync/sync.ts +9 -3
- package/src/sync/util.ts +10 -0
- package/src/util/config/collectors/config-collector.ts +16 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +5 -5
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +4 -4
- package/src/util/config/types.ts +1 -2
- package/src/util/env.ts +5 -0
- package/test/src/checksum_cache.test.ts +102 -57
- package/test/src/config.test.ts +115 -0
- package/test/src/routes/admin.test.ts +48 -0
- package/test/src/routes/mocks.ts +22 -1
- package/test/src/routes/stream.test.ts +3 -2
- package/test/src/sync/BucketChecksumState.test.ts +332 -93
- package/test/src/utils.ts +9 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -13,11 +13,44 @@ import {
|
|
|
13
13
|
WatchFilterEvent
|
|
14
14
|
} from '@/index.js';
|
|
15
15
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
16
|
-
import {
|
|
17
|
-
|
|
16
|
+
import {
|
|
17
|
+
ParameterIndexLookupCreator,
|
|
18
|
+
RequestJwtPayload,
|
|
19
|
+
ScopedParameterLookup,
|
|
20
|
+
SqliteJsonRow,
|
|
21
|
+
SqliteRow,
|
|
22
|
+
SqlSyncRules,
|
|
23
|
+
TablePattern,
|
|
24
|
+
SourceTableInterface,
|
|
25
|
+
versionedHydrationState
|
|
26
|
+
} from '@powersync/service-sync-rules';
|
|
27
|
+
import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
|
|
18
28
|
import { beforeEach, describe, expect, test } from 'vitest';
|
|
19
29
|
|
|
20
30
|
describe('BucketChecksumState', () => {
|
|
31
|
+
const LOOKUP_SOURCE: ParameterIndexLookupCreator = {
|
|
32
|
+
get defaultLookupScope(): ParameterLookupScope {
|
|
33
|
+
return {
|
|
34
|
+
lookupName: 'lookup',
|
|
35
|
+
queryId: '0',
|
|
36
|
+
source: LOOKUP_SOURCE
|
|
37
|
+
};
|
|
38
|
+
},
|
|
39
|
+
getSourceTables(): Set<TablePattern> {
|
|
40
|
+
return new Set();
|
|
41
|
+
},
|
|
42
|
+
evaluateParameterRow(_sourceTable: SourceTableInterface, _row: SqliteRow) {
|
|
43
|
+
return [];
|
|
44
|
+
},
|
|
45
|
+
tableSyncsParameters(_table: SourceTableInterface): boolean {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
function lookupScope(lookupName: string, queryId: string): ParameterLookupScope {
|
|
51
|
+
return { lookupName, queryId, source: LOOKUP_SOURCE };
|
|
52
|
+
}
|
|
53
|
+
|
|
21
54
|
// Single global[] bucket.
|
|
22
55
|
// We don't care about data in these tests
|
|
23
56
|
const SYNC_RULES_GLOBAL = SqlSyncRules.fromYaml(
|
|
@@ -62,10 +95,14 @@ bucket_definitions:
|
|
|
62
95
|
const syncRequest: StreamingSyncRequest = {};
|
|
63
96
|
const tokenPayload = new JwtPayload({ sub: '' });
|
|
64
97
|
|
|
98
|
+
function bucketStarts(requests: { bucket: string; start: InternalOpId }[]) {
|
|
99
|
+
return new Map(requests.map((request) => [request.bucket, request.start]));
|
|
100
|
+
}
|
|
101
|
+
|
|
65
102
|
test('global bucket with update', async () => {
|
|
66
103
|
const storage = new MockBucketChecksumStateStorage();
|
|
67
104
|
// Set intial state
|
|
68
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
105
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
69
106
|
|
|
70
107
|
const state = new BucketChecksumState({
|
|
71
108
|
syncContext,
|
|
@@ -83,34 +120,34 @@ bucket_definitions:
|
|
|
83
120
|
line.advance();
|
|
84
121
|
expect(line.checkpointLine).toEqual({
|
|
85
122
|
checkpoint: {
|
|
86
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
123
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
87
124
|
last_op_id: '1',
|
|
88
125
|
write_checkpoint: undefined,
|
|
89
126
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
90
127
|
}
|
|
91
128
|
});
|
|
92
|
-
expect(line.bucketsToFetch).
|
|
129
|
+
expect(line.bucketsToFetch).toMatchObject([
|
|
93
130
|
{
|
|
94
|
-
bucket: 'global[]',
|
|
131
|
+
bucket: '1#global[]',
|
|
95
132
|
priority: 3
|
|
96
133
|
}
|
|
97
134
|
]);
|
|
98
135
|
// This is the bucket data to be fetched
|
|
99
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
|
|
136
|
+
expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
|
|
100
137
|
|
|
101
138
|
// This similuates the bucket data being sent
|
|
102
139
|
line.advance();
|
|
103
|
-
line.updateBucketPosition({ bucket: 'global[]', nextAfter: 1n, hasMore: false });
|
|
140
|
+
line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: false });
|
|
104
141
|
|
|
105
142
|
// Update bucket storage state
|
|
106
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 2, count: 2 });
|
|
143
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
|
|
107
144
|
|
|
108
145
|
// Now we get a new line
|
|
109
146
|
const line2 = (await state.buildNextCheckpointLine({
|
|
110
147
|
base: storage.makeCheckpoint(2n),
|
|
111
148
|
writeCheckpoint: null,
|
|
112
149
|
update: {
|
|
113
|
-
updatedDataBuckets: new Set(['global[]']),
|
|
150
|
+
updatedDataBuckets: new Set(['1#global[]']),
|
|
114
151
|
invalidateDataBuckets: false,
|
|
115
152
|
updatedParameterLookups: new Set(),
|
|
116
153
|
invalidateParameterBuckets: false
|
|
@@ -120,12 +157,14 @@ bucket_definitions:
|
|
|
120
157
|
expect(line2.checkpointLine).toEqual({
|
|
121
158
|
checkpoint_diff: {
|
|
122
159
|
removed_buckets: [],
|
|
123
|
-
updated_buckets: [
|
|
160
|
+
updated_buckets: [
|
|
161
|
+
{ bucket: '1#global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
162
|
+
],
|
|
124
163
|
last_op_id: '2',
|
|
125
164
|
write_checkpoint: undefined
|
|
126
165
|
}
|
|
127
166
|
});
|
|
128
|
-
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
|
|
167
|
+
expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
|
|
129
168
|
});
|
|
130
169
|
|
|
131
170
|
test('global bucket with initial state', async () => {
|
|
@@ -134,13 +173,13 @@ bucket_definitions:
|
|
|
134
173
|
/// (getFilteredBucketStates)
|
|
135
174
|
const storage = new MockBucketChecksumStateStorage();
|
|
136
175
|
// Set intial state
|
|
137
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
176
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
138
177
|
|
|
139
178
|
const state = new BucketChecksumState({
|
|
140
179
|
syncContext,
|
|
141
180
|
tokenPayload,
|
|
142
181
|
// Client sets the initial state here
|
|
143
|
-
syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
|
|
182
|
+
syncRequest: { buckets: [{ name: '1#global[]', after: '1' }] },
|
|
144
183
|
syncRules: SYNC_RULES_GLOBAL,
|
|
145
184
|
bucketStorage: storage
|
|
146
185
|
});
|
|
@@ -153,27 +192,27 @@ bucket_definitions:
|
|
|
153
192
|
line.advance();
|
|
154
193
|
expect(line.checkpointLine).toEqual({
|
|
155
194
|
checkpoint: {
|
|
156
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
195
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
157
196
|
last_op_id: '1',
|
|
158
197
|
write_checkpoint: undefined,
|
|
159
198
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
160
199
|
}
|
|
161
200
|
});
|
|
162
|
-
expect(line.bucketsToFetch).
|
|
201
|
+
expect(line.bucketsToFetch).toMatchObject([
|
|
163
202
|
{
|
|
164
|
-
bucket: 'global[]',
|
|
203
|
+
bucket: '1#global[]',
|
|
165
204
|
priority: 3
|
|
166
205
|
}
|
|
167
206
|
]);
|
|
168
207
|
// This is the main difference between this and the previous test
|
|
169
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
|
|
208
|
+
expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
|
|
170
209
|
});
|
|
171
210
|
|
|
172
211
|
test('multiple static buckets', async () => {
|
|
173
212
|
const storage = new MockBucketChecksumStateStorage();
|
|
174
213
|
// Set intial state
|
|
175
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
176
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
214
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
215
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
177
216
|
|
|
178
217
|
const state = new BucketChecksumState({
|
|
179
218
|
syncContext,
|
|
@@ -191,35 +230,35 @@ bucket_definitions:
|
|
|
191
230
|
expect(line.checkpointLine).toEqual({
|
|
192
231
|
checkpoint: {
|
|
193
232
|
buckets: [
|
|
194
|
-
{ bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
195
|
-
{ bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
233
|
+
{ bucket: '2#global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
234
|
+
{ bucket: '2#global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
196
235
|
],
|
|
197
236
|
last_op_id: '1',
|
|
198
237
|
write_checkpoint: undefined,
|
|
199
238
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
200
239
|
}
|
|
201
240
|
});
|
|
202
|
-
expect(line.bucketsToFetch).
|
|
241
|
+
expect(line.bucketsToFetch).toMatchObject([
|
|
203
242
|
{
|
|
204
|
-
bucket: 'global[1]',
|
|
243
|
+
bucket: '2#global[1]',
|
|
205
244
|
priority: 3
|
|
206
245
|
},
|
|
207
246
|
{
|
|
208
|
-
bucket: 'global[2]',
|
|
247
|
+
bucket: '2#global[2]',
|
|
209
248
|
priority: 3
|
|
210
249
|
}
|
|
211
250
|
]);
|
|
212
251
|
line.advance();
|
|
213
252
|
|
|
214
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
215
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
253
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
254
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
216
255
|
|
|
217
256
|
const line2 = (await state.buildNextCheckpointLine({
|
|
218
257
|
base: storage.makeCheckpoint(2n),
|
|
219
258
|
writeCheckpoint: null,
|
|
220
259
|
update: {
|
|
221
260
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
222
|
-
updatedDataBuckets: new Set(['global[1]', 'global[2]']),
|
|
261
|
+
updatedDataBuckets: new Set(['2#global[1]', '2#global[2]']),
|
|
223
262
|
invalidateDataBuckets: false
|
|
224
263
|
}
|
|
225
264
|
}))!;
|
|
@@ -227,8 +266,8 @@ bucket_definitions:
|
|
|
227
266
|
checkpoint_diff: {
|
|
228
267
|
removed_buckets: [],
|
|
229
268
|
updated_buckets: [
|
|
230
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
231
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
269
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
270
|
+
{ bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
232
271
|
],
|
|
233
272
|
last_op_id: '2',
|
|
234
273
|
write_checkpoint: undefined
|
|
@@ -251,7 +290,7 @@ bucket_definitions:
|
|
|
251
290
|
bucketStorage: storage
|
|
252
291
|
});
|
|
253
292
|
|
|
254
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
293
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
255
294
|
|
|
256
295
|
const line = (await state.buildNextCheckpointLine({
|
|
257
296
|
base: storage.makeCheckpoint(1n),
|
|
@@ -261,19 +300,19 @@ bucket_definitions:
|
|
|
261
300
|
line.advance();
|
|
262
301
|
expect(line.checkpointLine).toEqual({
|
|
263
302
|
checkpoint: {
|
|
264
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
303
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
265
304
|
last_op_id: '1',
|
|
266
305
|
write_checkpoint: undefined,
|
|
267
306
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
268
307
|
}
|
|
269
308
|
});
|
|
270
|
-
expect(line.bucketsToFetch).
|
|
309
|
+
expect(line.bucketsToFetch).toMatchObject([
|
|
271
310
|
{
|
|
272
|
-
bucket: 'global[]',
|
|
311
|
+
bucket: '1#global[]',
|
|
273
312
|
priority: 3
|
|
274
313
|
}
|
|
275
314
|
]);
|
|
276
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
|
|
315
|
+
expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
|
|
277
316
|
});
|
|
278
317
|
|
|
279
318
|
test('invalidating individual bucket', async () => {
|
|
@@ -281,8 +320,8 @@ bucket_definitions:
|
|
|
281
320
|
|
|
282
321
|
const storage = new MockBucketChecksumStateStorage();
|
|
283
322
|
// Set initial state
|
|
284
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
285
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
323
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
324
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
286
325
|
|
|
287
326
|
const state = new BucketChecksumState({
|
|
288
327
|
syncContext,
|
|
@@ -301,11 +340,11 @@ bucket_definitions:
|
|
|
301
340
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
302
341
|
});
|
|
303
342
|
line!.advance();
|
|
304
|
-
line!.updateBucketPosition({ bucket: 'global[1]', nextAfter: 1n, hasMore: false });
|
|
305
|
-
line!.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: false });
|
|
343
|
+
line!.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 1n, hasMore: false });
|
|
344
|
+
line!.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: false });
|
|
306
345
|
|
|
307
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
308
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
346
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
347
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
309
348
|
|
|
310
349
|
const line2 = (await state.buildNextCheckpointLine({
|
|
311
350
|
base: storage.makeCheckpoint(2n),
|
|
@@ -315,7 +354,7 @@ bucket_definitions:
|
|
|
315
354
|
// Invalidate the state for global[1] - will only re-check the single bucket.
|
|
316
355
|
// This is essentially inconsistent state, but is the simplest way to test that
|
|
317
356
|
// the filter is working.
|
|
318
|
-
updatedDataBuckets: new Set(['global[1]']),
|
|
357
|
+
updatedDataBuckets: new Set(['2#global[1]']),
|
|
319
358
|
invalidateDataBuckets: false
|
|
320
359
|
}
|
|
321
360
|
}))!;
|
|
@@ -324,13 +363,13 @@ bucket_definitions:
|
|
|
324
363
|
removed_buckets: [],
|
|
325
364
|
updated_buckets: [
|
|
326
365
|
// This does not include global[2], since it was not invalidated.
|
|
327
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
366
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
328
367
|
],
|
|
329
368
|
last_op_id: '2',
|
|
330
369
|
write_checkpoint: undefined
|
|
331
370
|
}
|
|
332
371
|
});
|
|
333
|
-
expect(line2.bucketsToFetch).
|
|
372
|
+
expect(line2.bucketsToFetch).toMatchObject([{ bucket: '2#global[1]', priority: 3 }]);
|
|
334
373
|
});
|
|
335
374
|
|
|
336
375
|
test('invalidating all buckets', async () => {
|
|
@@ -349,8 +388,8 @@ bucket_definitions:
|
|
|
349
388
|
// storage.filter = state.checkpointFilter;
|
|
350
389
|
|
|
351
390
|
// Set initial state
|
|
352
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
353
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
391
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
392
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
354
393
|
|
|
355
394
|
const line = await state.buildNextCheckpointLine({
|
|
356
395
|
base: storage.makeCheckpoint(1n),
|
|
@@ -360,8 +399,8 @@ bucket_definitions:
|
|
|
360
399
|
|
|
361
400
|
line!.advance();
|
|
362
401
|
|
|
363
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
364
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
402
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
403
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
365
404
|
|
|
366
405
|
const line2 = (await state.buildNextCheckpointLine({
|
|
367
406
|
base: storage.makeCheckpoint(2n),
|
|
@@ -373,24 +412,24 @@ bucket_definitions:
|
|
|
373
412
|
checkpoint_diff: {
|
|
374
413
|
removed_buckets: [],
|
|
375
414
|
updated_buckets: [
|
|
376
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
377
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
415
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
416
|
+
{ bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
378
417
|
],
|
|
379
418
|
last_op_id: '2',
|
|
380
419
|
write_checkpoint: undefined
|
|
381
420
|
}
|
|
382
421
|
});
|
|
383
|
-
expect(line2.bucketsToFetch).
|
|
384
|
-
{ bucket: 'global[1]', priority: 3 },
|
|
385
|
-
{ bucket: 'global[2]', priority: 3 }
|
|
422
|
+
expect(line2.bucketsToFetch).toMatchObject([
|
|
423
|
+
{ bucket: '2#global[1]', priority: 3 },
|
|
424
|
+
{ bucket: '2#global[2]', priority: 3 }
|
|
386
425
|
]);
|
|
387
426
|
});
|
|
388
427
|
|
|
389
428
|
test('interrupt and resume static buckets checkpoint', async () => {
|
|
390
429
|
const storage = new MockBucketChecksumStateStorage();
|
|
391
430
|
// Set intial state
|
|
392
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 3, count: 3 });
|
|
393
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 3, count: 3 });
|
|
431
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 3, count: 3 });
|
|
432
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 3, count: 3 });
|
|
394
433
|
|
|
395
434
|
const state = new BucketChecksumState({
|
|
396
435
|
syncContext,
|
|
@@ -409,39 +448,39 @@ bucket_definitions:
|
|
|
409
448
|
expect(line.checkpointLine).toEqual({
|
|
410
449
|
checkpoint: {
|
|
411
450
|
buckets: [
|
|
412
|
-
{ bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
413
|
-
{ bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
451
|
+
{ bucket: '2#global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
452
|
+
{ bucket: '2#global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
414
453
|
],
|
|
415
454
|
last_op_id: '3',
|
|
416
455
|
write_checkpoint: undefined,
|
|
417
456
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
418
457
|
}
|
|
419
458
|
});
|
|
420
|
-
expect(line.bucketsToFetch).
|
|
459
|
+
expect(line.bucketsToFetch).toMatchObject([
|
|
421
460
|
{
|
|
422
|
-
bucket: 'global[1]',
|
|
461
|
+
bucket: '2#global[1]',
|
|
423
462
|
priority: 3
|
|
424
463
|
},
|
|
425
464
|
{
|
|
426
|
-
bucket: 'global[2]',
|
|
465
|
+
bucket: '2#global[2]',
|
|
427
466
|
priority: 3
|
|
428
467
|
}
|
|
429
468
|
]);
|
|
430
469
|
|
|
431
470
|
// This is the bucket data to be fetched
|
|
432
|
-
expect(line.getFilteredBucketPositions()).toEqual(
|
|
471
|
+
expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(
|
|
433
472
|
new Map([
|
|
434
|
-
['global[1]', 0n],
|
|
435
|
-
['global[2]', 0n]
|
|
473
|
+
['2#global[1]', 0n],
|
|
474
|
+
['2#global[2]', 0n]
|
|
436
475
|
])
|
|
437
476
|
);
|
|
438
477
|
|
|
439
478
|
// No data changes here.
|
|
440
479
|
// We simulate partial data sent, before a checkpoint is interrupted.
|
|
441
480
|
line.advance();
|
|
442
|
-
line.updateBucketPosition({ bucket: 'global[1]', nextAfter: 3n, hasMore: false });
|
|
443
|
-
line.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: true });
|
|
444
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
|
|
481
|
+
line.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 3n, hasMore: false });
|
|
482
|
+
line.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: true });
|
|
483
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 4, count: 4 });
|
|
445
484
|
|
|
446
485
|
const line2 = (await state.buildNextCheckpointLine({
|
|
447
486
|
base: storage.makeCheckpoint(4n),
|
|
@@ -449,7 +488,7 @@ bucket_definitions:
|
|
|
449
488
|
update: {
|
|
450
489
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
451
490
|
invalidateDataBuckets: false,
|
|
452
|
-
updatedDataBuckets: new Set(['global[1]'])
|
|
491
|
+
updatedDataBuckets: new Set(['2#global[1]'])
|
|
453
492
|
}
|
|
454
493
|
}))!;
|
|
455
494
|
line2.advance();
|
|
@@ -458,7 +497,7 @@ bucket_definitions:
|
|
|
458
497
|
removed_buckets: [],
|
|
459
498
|
updated_buckets: [
|
|
460
499
|
{
|
|
461
|
-
bucket: 'global[1]',
|
|
500
|
+
bucket: '2#global[1]',
|
|
462
501
|
checksum: 4,
|
|
463
502
|
count: 4,
|
|
464
503
|
priority: 3,
|
|
@@ -470,21 +509,21 @@ bucket_definitions:
|
|
|
470
509
|
}
|
|
471
510
|
});
|
|
472
511
|
// This should contain both buckets, even though only one changed.
|
|
473
|
-
expect(line2.bucketsToFetch).
|
|
512
|
+
expect(line2.bucketsToFetch).toMatchObject([
|
|
474
513
|
{
|
|
475
|
-
bucket: 'global[1]',
|
|
514
|
+
bucket: '2#global[1]',
|
|
476
515
|
priority: 3
|
|
477
516
|
},
|
|
478
517
|
{
|
|
479
|
-
bucket: 'global[2]',
|
|
518
|
+
bucket: '2#global[2]',
|
|
480
519
|
priority: 3
|
|
481
520
|
}
|
|
482
521
|
]);
|
|
483
522
|
|
|
484
|
-
expect(line2.getFilteredBucketPositions()).toEqual(
|
|
523
|
+
expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(
|
|
485
524
|
new Map([
|
|
486
|
-
['global[1]', 3n],
|
|
487
|
-
['global[2]', 1n]
|
|
525
|
+
['2#global[1]', 3n],
|
|
526
|
+
['2#global[2]', 1n]
|
|
488
527
|
])
|
|
489
528
|
);
|
|
490
529
|
});
|
|
@@ -492,9 +531,9 @@ bucket_definitions:
|
|
|
492
531
|
test('dynamic buckets with updates', async () => {
|
|
493
532
|
const storage = new MockBucketChecksumStateStorage();
|
|
494
533
|
// Set intial state
|
|
495
|
-
storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
|
|
496
|
-
storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
|
|
497
|
-
storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
|
|
534
|
+
storage.updateTestChecksum({ bucket: '3#by_project[1]', checksum: 1, count: 1 });
|
|
535
|
+
storage.updateTestChecksum({ bucket: '3#by_project[2]', checksum: 1, count: 1 });
|
|
536
|
+
storage.updateTestChecksum({ bucket: '3#by_project[3]', checksum: 1, count: 1 });
|
|
498
537
|
|
|
499
538
|
const state = new BucketChecksumState({
|
|
500
539
|
syncContext,
|
|
@@ -506,7 +545,7 @@ bucket_definitions:
|
|
|
506
545
|
|
|
507
546
|
const line = (await state.buildNextCheckpointLine({
|
|
508
547
|
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
509
|
-
expect(lookups).toEqual([ScopedParameterLookup.direct(
|
|
548
|
+
expect(lookups).toEqual([ScopedParameterLookup.direct(lookupScope('by_project', '1'), ['u1'])]);
|
|
510
549
|
return [{ id: 1 }, { id: 2 }];
|
|
511
550
|
}),
|
|
512
551
|
writeCheckpoint: null,
|
|
@@ -516,14 +555,14 @@ bucket_definitions:
|
|
|
516
555
|
checkpoint: {
|
|
517
556
|
buckets: [
|
|
518
557
|
{
|
|
519
|
-
bucket: 'by_project[1]',
|
|
558
|
+
bucket: '3#by_project[1]',
|
|
520
559
|
checksum: 1,
|
|
521
560
|
count: 1,
|
|
522
561
|
priority: 3,
|
|
523
562
|
subscriptions: [{ default: 0 }]
|
|
524
563
|
},
|
|
525
564
|
{
|
|
526
|
-
bucket: 'by_project[2]',
|
|
565
|
+
bucket: '3#by_project[2]',
|
|
527
566
|
checksum: 1,
|
|
528
567
|
count: 1,
|
|
529
568
|
priority: 3,
|
|
@@ -541,33 +580,33 @@ bucket_definitions:
|
|
|
541
580
|
write_checkpoint: undefined
|
|
542
581
|
}
|
|
543
582
|
});
|
|
544
|
-
expect(line.bucketsToFetch).
|
|
583
|
+
expect(line.bucketsToFetch).toMatchObject([
|
|
545
584
|
{
|
|
546
|
-
bucket: 'by_project[1]',
|
|
585
|
+
bucket: '3#by_project[1]',
|
|
547
586
|
priority: 3
|
|
548
587
|
},
|
|
549
588
|
{
|
|
550
|
-
bucket: 'by_project[2]',
|
|
589
|
+
bucket: '3#by_project[2]',
|
|
551
590
|
priority: 3
|
|
552
591
|
}
|
|
553
592
|
]);
|
|
554
593
|
line.advance();
|
|
555
594
|
// This is the bucket data to be fetched
|
|
556
|
-
expect(line.getFilteredBucketPositions()).toEqual(
|
|
595
|
+
expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(
|
|
557
596
|
new Map([
|
|
558
|
-
['by_project[1]', 0n],
|
|
559
|
-
['by_project[2]', 0n]
|
|
597
|
+
['3#by_project[1]', 0n],
|
|
598
|
+
['3#by_project[2]', 0n]
|
|
560
599
|
])
|
|
561
600
|
);
|
|
562
601
|
|
|
563
602
|
line.advance();
|
|
564
|
-
line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
|
|
565
|
-
line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
|
|
603
|
+
line.updateBucketPosition({ bucket: '3#by_project[1]', nextAfter: 1n, hasMore: false });
|
|
604
|
+
line.updateBucketPosition({ bucket: '3#by_project[2]', nextAfter: 1n, hasMore: false });
|
|
566
605
|
|
|
567
606
|
// Now we get a new line
|
|
568
607
|
const line2 = (await state.buildNextCheckpointLine({
|
|
569
608
|
base: storage.makeCheckpoint(2n, (lookups) => {
|
|
570
|
-
expect(lookups).toEqual([ScopedParameterLookup.direct(
|
|
609
|
+
expect(lookups).toEqual([ScopedParameterLookup.direct(lookupScope('by_project', '1'), ['u1'])]);
|
|
571
610
|
return [{ id: 1 }, { id: 2 }, { id: 3 }];
|
|
572
611
|
}),
|
|
573
612
|
writeCheckpoint: null,
|
|
@@ -584,7 +623,7 @@ bucket_definitions:
|
|
|
584
623
|
removed_buckets: [],
|
|
585
624
|
updated_buckets: [
|
|
586
625
|
{
|
|
587
|
-
bucket: 'by_project[3]',
|
|
626
|
+
bucket: '3#by_project[3]',
|
|
588
627
|
checksum: 1,
|
|
589
628
|
count: 1,
|
|
590
629
|
priority: 3,
|
|
@@ -595,7 +634,7 @@ bucket_definitions:
|
|
|
595
634
|
write_checkpoint: undefined
|
|
596
635
|
}
|
|
597
636
|
});
|
|
598
|
-
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
|
|
637
|
+
expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['3#by_project[3]', 0n]]));
|
|
599
638
|
});
|
|
600
639
|
|
|
601
640
|
describe('streams', () => {
|
|
@@ -820,6 +859,206 @@ config:
|
|
|
820
859
|
});
|
|
821
860
|
});
|
|
822
861
|
});
|
|
862
|
+
|
|
863
|
+
describe('parameter query result logging', () => {
|
|
864
|
+
test('logs parameter query results for dynamic buckets', async () => {
|
|
865
|
+
const storage = new MockBucketChecksumStateStorage();
|
|
866
|
+
storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
|
|
867
|
+
storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
|
|
868
|
+
storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
|
|
869
|
+
|
|
870
|
+
const logMessages: string[] = [];
|
|
871
|
+
const logData: any[] = [];
|
|
872
|
+
const mockLogger = {
|
|
873
|
+
info: (message: string, data: any) => {
|
|
874
|
+
logMessages.push(message);
|
|
875
|
+
logData.push(data);
|
|
876
|
+
},
|
|
877
|
+
error: () => {},
|
|
878
|
+
warn: () => {},
|
|
879
|
+
debug: () => {}
|
|
880
|
+
};
|
|
881
|
+
|
|
882
|
+
const state = new BucketChecksumState({
|
|
883
|
+
syncContext,
|
|
884
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
885
|
+
syncRequest,
|
|
886
|
+
syncRules: SYNC_RULES_DYNAMIC,
|
|
887
|
+
bucketStorage: storage,
|
|
888
|
+
logger: mockLogger as any
|
|
889
|
+
});
|
|
890
|
+
|
|
891
|
+
const line = (await state.buildNextCheckpointLine({
|
|
892
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
893
|
+
return [{ id: 1 }, { id: 2 }, { id: 3 }];
|
|
894
|
+
}),
|
|
895
|
+
writeCheckpoint: null,
|
|
896
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
897
|
+
}))!;
|
|
898
|
+
line.advance();
|
|
899
|
+
|
|
900
|
+
expect(logMessages[0]).toContain('param_results: 3');
|
|
901
|
+
expect(logData[0].parameter_query_results).toBe(3);
|
|
902
|
+
});
|
|
903
|
+
|
|
904
|
+
test('throws error with breakdown when parameter query limit is exceeded', async () => {
|
|
905
|
+
const SYNC_RULES_MULTI = SqlSyncRules.fromYaml(
|
|
906
|
+
`
|
|
907
|
+
bucket_definitions:
|
|
908
|
+
projects:
|
|
909
|
+
parameters: select id from projects where user_id = request.user_id()
|
|
910
|
+
data: []
|
|
911
|
+
tasks:
|
|
912
|
+
parameters: select id from tasks where user_id = request.user_id()
|
|
913
|
+
data: []
|
|
914
|
+
comments:
|
|
915
|
+
parameters: select id from comments where user_id = request.user_id()
|
|
916
|
+
data: []
|
|
917
|
+
`,
|
|
918
|
+
{ defaultSchema: 'public' }
|
|
919
|
+
).config.hydrate({ hydrationState: versionedHydrationState(4) });
|
|
920
|
+
|
|
921
|
+
const storage = new MockBucketChecksumStateStorage();
|
|
922
|
+
|
|
923
|
+
const errorMessages: string[] = [];
|
|
924
|
+
const errorData: any[] = [];
|
|
925
|
+
const mockLogger = {
|
|
926
|
+
info: () => {},
|
|
927
|
+
error: (message: string, data: any) => {
|
|
928
|
+
errorMessages.push(message);
|
|
929
|
+
errorData.push(data);
|
|
930
|
+
},
|
|
931
|
+
warn: () => {},
|
|
932
|
+
debug: () => {}
|
|
933
|
+
};
|
|
934
|
+
|
|
935
|
+
const smallContext = new SyncContext({
|
|
936
|
+
maxBuckets: 100,
|
|
937
|
+
maxParameterQueryResults: 50,
|
|
938
|
+
maxDataFetchConcurrency: 10
|
|
939
|
+
});
|
|
940
|
+
|
|
941
|
+
const state = new BucketChecksumState({
|
|
942
|
+
syncContext: smallContext,
|
|
943
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
944
|
+
syncRequest,
|
|
945
|
+
syncRules: SYNC_RULES_MULTI,
|
|
946
|
+
bucketStorage: storage,
|
|
947
|
+
logger: mockLogger as any
|
|
948
|
+
});
|
|
949
|
+
|
|
950
|
+
// Create 60 total results: 30 projects + 20 tasks + 10 comments
|
|
951
|
+
const projectIds = Array.from({ length: 30 }, (_, i) => ({ id: i + 1 }));
|
|
952
|
+
const taskIds = Array.from({ length: 20 }, (_, i) => ({ id: i + 1 }));
|
|
953
|
+
const commentIds = Array.from({ length: 10 }, (_, i) => ({ id: i + 1 }));
|
|
954
|
+
|
|
955
|
+
for (let i = 1; i <= 30; i++) {
|
|
956
|
+
storage.updateTestChecksum({ bucket: `projects[${i}]`, checksum: 1, count: 1 });
|
|
957
|
+
}
|
|
958
|
+
for (let i = 1; i <= 20; i++) {
|
|
959
|
+
storage.updateTestChecksum({ bucket: `tasks[${i}]`, checksum: 1, count: 1 });
|
|
960
|
+
}
|
|
961
|
+
for (let i = 1; i <= 10; i++) {
|
|
962
|
+
storage.updateTestChecksum({ bucket: `comments[${i}]`, checksum: 1, count: 1 });
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
await expect(
|
|
966
|
+
state.buildNextCheckpointLine({
|
|
967
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
968
|
+
const lookup = lookups[0];
|
|
969
|
+
const lookupName = lookup.values[0];
|
|
970
|
+
if (lookupName === 'projects') {
|
|
971
|
+
return projectIds;
|
|
972
|
+
} else if (lookupName === 'tasks') {
|
|
973
|
+
return taskIds;
|
|
974
|
+
} else {
|
|
975
|
+
return commentIds;
|
|
976
|
+
}
|
|
977
|
+
}),
|
|
978
|
+
writeCheckpoint: null,
|
|
979
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
980
|
+
})
|
|
981
|
+
).rejects.toThrow('Too many parameter query results: 60 (limit of 50)');
|
|
982
|
+
|
|
983
|
+
// Verify error log includes breakdown
|
|
984
|
+
expect(errorMessages[0]).toContain('Parameter query results by definition:');
|
|
985
|
+
expect(errorMessages[0]).toContain('projects: 30');
|
|
986
|
+
expect(errorMessages[0]).toContain('tasks: 20');
|
|
987
|
+
expect(errorMessages[0]).toContain('comments: 10');
|
|
988
|
+
|
|
989
|
+
expect(errorData[0].parameter_query_results).toBe(60);
|
|
990
|
+
expect(errorData[0].parameter_query_results_by_definition).toEqual({
|
|
991
|
+
projects: 30,
|
|
992
|
+
tasks: 20,
|
|
993
|
+
comments: 10
|
|
994
|
+
});
|
|
995
|
+
});
|
|
996
|
+
|
|
997
|
+
test('limits breakdown to top 10 definitions', async () => {
|
|
998
|
+
// Create sync rules with 15 different definitions with dynamic parameters
|
|
999
|
+
let yamlDefinitions = 'bucket_definitions:\n';
|
|
1000
|
+
for (let i = 1; i <= 15; i++) {
|
|
1001
|
+
yamlDefinitions += ` def${i}:\n`;
|
|
1002
|
+
yamlDefinitions += ` parameters: select id from def${i}_table where user_id = request.user_id()\n`;
|
|
1003
|
+
yamlDefinitions += ` data: []\n`;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
const SYNC_RULES_MANY = SqlSyncRules.fromYaml(yamlDefinitions, { defaultSchema: 'public' }).config.hydrate({
|
|
1007
|
+
hydrationState: versionedHydrationState(5)
|
|
1008
|
+
});
|
|
1009
|
+
|
|
1010
|
+
const storage = new MockBucketChecksumStateStorage();
|
|
1011
|
+
|
|
1012
|
+
const errorMessages: string[] = [];
|
|
1013
|
+
const mockLogger = {
|
|
1014
|
+
info: () => {},
|
|
1015
|
+
error: (message: string) => {
|
|
1016
|
+
errorMessages.push(message);
|
|
1017
|
+
},
|
|
1018
|
+
warn: () => {},
|
|
1019
|
+
debug: () => {}
|
|
1020
|
+
};
|
|
1021
|
+
|
|
1022
|
+
const smallContext = new SyncContext({
|
|
1023
|
+
maxBuckets: 100,
|
|
1024
|
+
maxParameterQueryResults: 10,
|
|
1025
|
+
maxDataFetchConcurrency: 10
|
|
1026
|
+
});
|
|
1027
|
+
|
|
1028
|
+
const state = new BucketChecksumState({
|
|
1029
|
+
syncContext: smallContext,
|
|
1030
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
1031
|
+
syncRequest,
|
|
1032
|
+
syncRules: SYNC_RULES_MANY,
|
|
1033
|
+
bucketStorage: storage,
|
|
1034
|
+
logger: mockLogger as any
|
|
1035
|
+
});
|
|
1036
|
+
|
|
1037
|
+
// Each definition creates one bucket, total 15 buckets
|
|
1038
|
+
for (let i = 1; i <= 15; i++) {
|
|
1039
|
+
storage.updateTestChecksum({ bucket: `def${i}[${i}]`, checksum: 1, count: 1 });
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
await expect(
|
|
1043
|
+
state.buildNextCheckpointLine({
|
|
1044
|
+
base: storage.makeCheckpoint(1n, (lookups) => {
|
|
1045
|
+
// Return one result for each definition
|
|
1046
|
+
return [{ id: 1 }];
|
|
1047
|
+
}),
|
|
1048
|
+
writeCheckpoint: null,
|
|
1049
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
1050
|
+
})
|
|
1051
|
+
).rejects.toThrow('Too many parameter query results: 15 (limit of 10)');
|
|
1052
|
+
|
|
1053
|
+
// Verify only top 10 are shown
|
|
1054
|
+
const errorMessage = errorMessages[0];
|
|
1055
|
+
expect(errorMessage).toContain('... and 5 more results from 5 definitions');
|
|
1056
|
+
|
|
1057
|
+
// Count how many definitions are listed (should be exactly 10)
|
|
1058
|
+
const defMatches = errorMessage.match(/def\d+:/g);
|
|
1059
|
+
expect(defMatches?.length).toBe(10);
|
|
1060
|
+
});
|
|
1061
|
+
});
|
|
823
1062
|
});
|
|
824
1063
|
|
|
825
1064
|
class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
|
|
@@ -837,9 +1076,9 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
|
|
|
837
1076
|
this.filter?.({ invalidate: true });
|
|
838
1077
|
}
|
|
839
1078
|
|
|
840
|
-
async getChecksums(
|
|
1079
|
+
async getChecksums(_checkpoint: InternalOpId, buckets: { bucket: string }[]): Promise<ChecksumMap> {
|
|
841
1080
|
return new Map<string, BucketChecksum>(
|
|
842
|
-
buckets.map((bucket) => {
|
|
1081
|
+
buckets.map(({ bucket }) => {
|
|
843
1082
|
const checksum = this.state.get(bucket);
|
|
844
1083
|
return [
|
|
845
1084
|
bucket,
|