@powersync/service-core 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223080959
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -8
- package/dist/api/RouteAPI.d.ts +2 -2
- package/dist/api/diagnostics.js +14 -6
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/JwtPayload.d.ts +7 -8
- package/dist/auth/JwtPayload.js +19 -1
- package/dist/auth/JwtPayload.js.map +1 -1
- package/dist/auth/KeyStore.js +2 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/auth.d.ts +0 -1
- package/dist/routes/auth.js +2 -4
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +84 -0
- package/dist/routes/configure-fastify.js +0 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +171 -0
- package/dist/routes/endpoints/admin.js +36 -21
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +3 -3
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +4 -10
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +10 -13
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +3 -8
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/router.d.ts +0 -1
- package/dist/routes/router.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +29 -15
- package/dist/storage/BucketStorageFactory.js +58 -1
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +28 -4
- package/dist/storage/PersistedSyncRulesContent.js +56 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReportStorage.d.ts +1 -8
- package/dist/storage/StorageVersionConfig.d.ts +20 -0
- package/dist/storage/StorageVersionConfig.js +20 -0
- package/dist/storage/StorageVersionConfig.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +8 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +4 -6
- package/dist/sync/BucketChecksumState.js +4 -9
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.d.ts +0 -8
- package/dist/sync/sync.js +9 -19
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +0 -22
- package/dist/sync/util.js +0 -24
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config.js +4 -1
- package/dist/util/config.js.map +1 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +2 -2
- package/src/api/diagnostics.ts +16 -7
- package/src/auth/JwtPayload.ts +16 -8
- package/src/auth/KeyStore.ts +1 -1
- package/src/replication/AbstractReplicator.ts +3 -5
- package/src/routes/auth.ts +2 -4
- package/src/routes/configure-fastify.ts +0 -1
- package/src/routes/endpoints/admin.ts +45 -26
- package/src/routes/endpoints/checkpointing.ts +5 -3
- package/src/routes/endpoints/socket-route.ts +4 -11
- package/src/routes/endpoints/sync-rules.ts +18 -17
- package/src/routes/endpoints/sync-stream.ts +4 -8
- package/src/routes/router.ts +0 -2
- package/src/storage/BucketStorageFactory.ts +67 -19
- package/src/storage/PersistedSyncRulesContent.ts +82 -5
- package/src/storage/ReportStorage.ts +3 -9
- package/src/storage/StorageVersionConfig.ts +30 -0
- package/src/storage/SyncRulesBucketStorage.ts +9 -0
- package/src/storage/storage-index.ts +1 -0
- package/src/sync/BucketChecksumState.ts +10 -13
- package/src/sync/sync.ts +15 -42
- package/src/sync/util.ts +0 -25
- package/src/util/config.ts +7 -2
- package/test/src/auth.test.ts +76 -20
- package/test/src/routes/admin.test.ts +48 -0
- package/test/src/routes/mocks.ts +22 -1
- package/test/src/routes/stream.test.ts +10 -9
- package/test/src/sync/BucketChecksumState.test.ts +92 -84
- package/test/tsconfig.json +3 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -6,14 +6,20 @@ import {
|
|
|
6
6
|
CHECKPOINT_INVALIDATE_ALL,
|
|
7
7
|
ChecksumMap,
|
|
8
8
|
InternalOpId,
|
|
9
|
+
JwtPayload,
|
|
9
10
|
ReplicationCheckpoint,
|
|
10
11
|
StreamingSyncRequest,
|
|
11
12
|
SyncContext,
|
|
12
13
|
WatchFilterEvent
|
|
13
14
|
} from '@/index.js';
|
|
14
15
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
15
|
-
import {
|
|
16
|
-
|
|
16
|
+
import {
|
|
17
|
+
RequestJwtPayload,
|
|
18
|
+
ScopedParameterLookup,
|
|
19
|
+
SqliteJsonRow,
|
|
20
|
+
SqlSyncRules,
|
|
21
|
+
versionedHydrationState
|
|
22
|
+
} from '@powersync/service-sync-rules';
|
|
17
23
|
import { beforeEach, describe, expect, test } from 'vitest';
|
|
18
24
|
|
|
19
25
|
describe('BucketChecksumState', () => {
|
|
@@ -26,7 +32,7 @@ bucket_definitions:
|
|
|
26
32
|
data: []
|
|
27
33
|
`,
|
|
28
34
|
{ defaultSchema: 'public' }
|
|
29
|
-
).hydrate({ hydrationState: versionedHydrationState(1) });
|
|
35
|
+
).config.hydrate({ hydrationState: versionedHydrationState(1) });
|
|
30
36
|
|
|
31
37
|
// global[1] and global[2]
|
|
32
38
|
const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml(
|
|
@@ -39,7 +45,7 @@ bucket_definitions:
|
|
|
39
45
|
data: []
|
|
40
46
|
`,
|
|
41
47
|
{ defaultSchema: 'public' }
|
|
42
|
-
).hydrate({ hydrationState: versionedHydrationState(2) });
|
|
48
|
+
).config.hydrate({ hydrationState: versionedHydrationState(2) });
|
|
43
49
|
|
|
44
50
|
// by_project[n]
|
|
45
51
|
const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml(
|
|
@@ -50,7 +56,7 @@ bucket_definitions:
|
|
|
50
56
|
data: []
|
|
51
57
|
`,
|
|
52
58
|
{ defaultSchema: 'public' }
|
|
53
|
-
).hydrate({ hydrationState: versionedHydrationState(3) });
|
|
59
|
+
).config.hydrate({ hydrationState: versionedHydrationState(3) });
|
|
54
60
|
|
|
55
61
|
const syncContext = new SyncContext({
|
|
56
62
|
maxBuckets: 100,
|
|
@@ -59,12 +65,12 @@ bucket_definitions:
|
|
|
59
65
|
});
|
|
60
66
|
|
|
61
67
|
const syncRequest: StreamingSyncRequest = {};
|
|
62
|
-
const tokenPayload
|
|
68
|
+
const tokenPayload = new JwtPayload({ sub: '' });
|
|
63
69
|
|
|
64
70
|
test('global bucket with update', async () => {
|
|
65
71
|
const storage = new MockBucketChecksumStateStorage();
|
|
66
72
|
// Set intial state
|
|
67
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
73
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
68
74
|
|
|
69
75
|
const state = new BucketChecksumState({
|
|
70
76
|
syncContext,
|
|
@@ -82,7 +88,7 @@ bucket_definitions:
|
|
|
82
88
|
line.advance();
|
|
83
89
|
expect(line.checkpointLine).toEqual({
|
|
84
90
|
checkpoint: {
|
|
85
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
91
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
86
92
|
last_op_id: '1',
|
|
87
93
|
write_checkpoint: undefined,
|
|
88
94
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
@@ -90,26 +96,26 @@ bucket_definitions:
|
|
|
90
96
|
});
|
|
91
97
|
expect(line.bucketsToFetch).toEqual([
|
|
92
98
|
{
|
|
93
|
-
bucket: 'global[]',
|
|
99
|
+
bucket: '1#global[]',
|
|
94
100
|
priority: 3
|
|
95
101
|
}
|
|
96
102
|
]);
|
|
97
103
|
// This is the bucket data to be fetched
|
|
98
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
|
|
104
|
+
expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
|
|
99
105
|
|
|
100
106
|
// This similuates the bucket data being sent
|
|
101
107
|
line.advance();
|
|
102
|
-
line.updateBucketPosition({ bucket: 'global[]', nextAfter: 1n, hasMore: false });
|
|
108
|
+
line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: false });
|
|
103
109
|
|
|
104
110
|
// Update bucket storage state
|
|
105
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 2, count: 2 });
|
|
111
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
|
|
106
112
|
|
|
107
113
|
// Now we get a new line
|
|
108
114
|
const line2 = (await state.buildNextCheckpointLine({
|
|
109
115
|
base: storage.makeCheckpoint(2n),
|
|
110
116
|
writeCheckpoint: null,
|
|
111
117
|
update: {
|
|
112
|
-
updatedDataBuckets: new Set(['global[]']),
|
|
118
|
+
updatedDataBuckets: new Set(['1#global[]']),
|
|
113
119
|
invalidateDataBuckets: false,
|
|
114
120
|
updatedParameterLookups: new Set(),
|
|
115
121
|
invalidateParameterBuckets: false
|
|
@@ -119,12 +125,14 @@ bucket_definitions:
|
|
|
119
125
|
expect(line2.checkpointLine).toEqual({
|
|
120
126
|
checkpoint_diff: {
|
|
121
127
|
removed_buckets: [],
|
|
122
|
-
updated_buckets: [
|
|
128
|
+
updated_buckets: [
|
|
129
|
+
{ bucket: '1#global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
130
|
+
],
|
|
123
131
|
last_op_id: '2',
|
|
124
132
|
write_checkpoint: undefined
|
|
125
133
|
}
|
|
126
134
|
});
|
|
127
|
-
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
|
|
135
|
+
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
|
|
128
136
|
});
|
|
129
137
|
|
|
130
138
|
test('global bucket with initial state', async () => {
|
|
@@ -133,13 +141,13 @@ bucket_definitions:
|
|
|
133
141
|
/// (getFilteredBucketStates)
|
|
134
142
|
const storage = new MockBucketChecksumStateStorage();
|
|
135
143
|
// Set intial state
|
|
136
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
144
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
137
145
|
|
|
138
146
|
const state = new BucketChecksumState({
|
|
139
147
|
syncContext,
|
|
140
148
|
tokenPayload,
|
|
141
149
|
// Client sets the initial state here
|
|
142
|
-
syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
|
|
150
|
+
syncRequest: { buckets: [{ name: '1#global[]', after: '1' }] },
|
|
143
151
|
syncRules: SYNC_RULES_GLOBAL,
|
|
144
152
|
bucketStorage: storage
|
|
145
153
|
});
|
|
@@ -152,7 +160,7 @@ bucket_definitions:
|
|
|
152
160
|
line.advance();
|
|
153
161
|
expect(line.checkpointLine).toEqual({
|
|
154
162
|
checkpoint: {
|
|
155
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
163
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
156
164
|
last_op_id: '1',
|
|
157
165
|
write_checkpoint: undefined,
|
|
158
166
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
@@ -160,19 +168,19 @@ bucket_definitions:
|
|
|
160
168
|
});
|
|
161
169
|
expect(line.bucketsToFetch).toEqual([
|
|
162
170
|
{
|
|
163
|
-
bucket: 'global[]',
|
|
171
|
+
bucket: '1#global[]',
|
|
164
172
|
priority: 3
|
|
165
173
|
}
|
|
166
174
|
]);
|
|
167
175
|
// This is the main difference between this and the previous test
|
|
168
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 1n]]));
|
|
176
|
+
expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
|
|
169
177
|
});
|
|
170
178
|
|
|
171
179
|
test('multiple static buckets', async () => {
|
|
172
180
|
const storage = new MockBucketChecksumStateStorage();
|
|
173
181
|
// Set intial state
|
|
174
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
175
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
182
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
183
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
176
184
|
|
|
177
185
|
const state = new BucketChecksumState({
|
|
178
186
|
syncContext,
|
|
@@ -190,8 +198,8 @@ bucket_definitions:
|
|
|
190
198
|
expect(line.checkpointLine).toEqual({
|
|
191
199
|
checkpoint: {
|
|
192
200
|
buckets: [
|
|
193
|
-
{ bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
194
|
-
{ bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
201
|
+
{ bucket: '2#global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
|
|
202
|
+
{ bucket: '2#global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
|
|
195
203
|
],
|
|
196
204
|
last_op_id: '1',
|
|
197
205
|
write_checkpoint: undefined,
|
|
@@ -200,25 +208,25 @@ bucket_definitions:
|
|
|
200
208
|
});
|
|
201
209
|
expect(line.bucketsToFetch).toEqual([
|
|
202
210
|
{
|
|
203
|
-
bucket: 'global[1]',
|
|
211
|
+
bucket: '2#global[1]',
|
|
204
212
|
priority: 3
|
|
205
213
|
},
|
|
206
214
|
{
|
|
207
|
-
bucket: 'global[2]',
|
|
215
|
+
bucket: '2#global[2]',
|
|
208
216
|
priority: 3
|
|
209
217
|
}
|
|
210
218
|
]);
|
|
211
219
|
line.advance();
|
|
212
220
|
|
|
213
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
214
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
221
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
222
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
215
223
|
|
|
216
224
|
const line2 = (await state.buildNextCheckpointLine({
|
|
217
225
|
base: storage.makeCheckpoint(2n),
|
|
218
226
|
writeCheckpoint: null,
|
|
219
227
|
update: {
|
|
220
228
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
221
|
-
updatedDataBuckets: new Set(['global[1]', 'global[2]']),
|
|
229
|
+
updatedDataBuckets: new Set(['2#global[1]', '2#global[2]']),
|
|
222
230
|
invalidateDataBuckets: false
|
|
223
231
|
}
|
|
224
232
|
}))!;
|
|
@@ -226,8 +234,8 @@ bucket_definitions:
|
|
|
226
234
|
checkpoint_diff: {
|
|
227
235
|
removed_buckets: [],
|
|
228
236
|
updated_buckets: [
|
|
229
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
230
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
237
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
238
|
+
{ bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
231
239
|
],
|
|
232
240
|
last_op_id: '2',
|
|
233
241
|
write_checkpoint: undefined
|
|
@@ -250,7 +258,7 @@ bucket_definitions:
|
|
|
250
258
|
bucketStorage: storage
|
|
251
259
|
});
|
|
252
260
|
|
|
253
|
-
storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
|
|
261
|
+
storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
|
|
254
262
|
|
|
255
263
|
const line = (await state.buildNextCheckpointLine({
|
|
256
264
|
base: storage.makeCheckpoint(1n),
|
|
@@ -260,7 +268,7 @@ bucket_definitions:
|
|
|
260
268
|
line.advance();
|
|
261
269
|
expect(line.checkpointLine).toEqual({
|
|
262
270
|
checkpoint: {
|
|
263
|
-
buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
271
|
+
buckets: [{ bucket: '1#global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
|
|
264
272
|
last_op_id: '1',
|
|
265
273
|
write_checkpoint: undefined,
|
|
266
274
|
streams: [{ name: 'global', is_default: true, errors: [] }]
|
|
@@ -268,11 +276,11 @@ bucket_definitions:
|
|
|
268
276
|
});
|
|
269
277
|
expect(line.bucketsToFetch).toEqual([
|
|
270
278
|
{
|
|
271
|
-
bucket: 'global[]',
|
|
279
|
+
bucket: '1#global[]',
|
|
272
280
|
priority: 3
|
|
273
281
|
}
|
|
274
282
|
]);
|
|
275
|
-
expect(line.getFilteredBucketPositions()).toEqual(new Map([['global[]', 0n]]));
|
|
283
|
+
expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
|
|
276
284
|
});
|
|
277
285
|
|
|
278
286
|
test('invalidating individual bucket', async () => {
|
|
@@ -280,8 +288,8 @@ bucket_definitions:
|
|
|
280
288
|
|
|
281
289
|
const storage = new MockBucketChecksumStateStorage();
|
|
282
290
|
// Set initial state
|
|
283
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
284
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
291
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
292
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
285
293
|
|
|
286
294
|
const state = new BucketChecksumState({
|
|
287
295
|
syncContext,
|
|
@@ -300,11 +308,11 @@ bucket_definitions:
|
|
|
300
308
|
update: CHECKPOINT_INVALIDATE_ALL
|
|
301
309
|
});
|
|
302
310
|
line!.advance();
|
|
303
|
-
line!.updateBucketPosition({ bucket: 'global[1]', nextAfter: 1n, hasMore: false });
|
|
304
|
-
line!.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: false });
|
|
311
|
+
line!.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 1n, hasMore: false });
|
|
312
|
+
line!.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: false });
|
|
305
313
|
|
|
306
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
307
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
314
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
315
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
308
316
|
|
|
309
317
|
const line2 = (await state.buildNextCheckpointLine({
|
|
310
318
|
base: storage.makeCheckpoint(2n),
|
|
@@ -314,7 +322,7 @@ bucket_definitions:
|
|
|
314
322
|
// Invalidate the state for global[1] - will only re-check the single bucket.
|
|
315
323
|
// This is essentially inconsistent state, but is the simplest way to test that
|
|
316
324
|
// the filter is working.
|
|
317
|
-
updatedDataBuckets: new Set(['global[1]']),
|
|
325
|
+
updatedDataBuckets: new Set(['2#global[1]']),
|
|
318
326
|
invalidateDataBuckets: false
|
|
319
327
|
}
|
|
320
328
|
}))!;
|
|
@@ -323,13 +331,13 @@ bucket_definitions:
|
|
|
323
331
|
removed_buckets: [],
|
|
324
332
|
updated_buckets: [
|
|
325
333
|
// This does not include global[2], since it was not invalidated.
|
|
326
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
334
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
327
335
|
],
|
|
328
336
|
last_op_id: '2',
|
|
329
337
|
write_checkpoint: undefined
|
|
330
338
|
}
|
|
331
339
|
});
|
|
332
|
-
expect(line2.bucketsToFetch).toEqual([{ bucket: 'global[1]', priority: 3 }]);
|
|
340
|
+
expect(line2.bucketsToFetch).toEqual([{ bucket: '2#global[1]', priority: 3 }]);
|
|
333
341
|
});
|
|
334
342
|
|
|
335
343
|
test('invalidating all buckets', async () => {
|
|
@@ -348,8 +356,8 @@ bucket_definitions:
|
|
|
348
356
|
// storage.filter = state.checkpointFilter;
|
|
349
357
|
|
|
350
358
|
// Set initial state
|
|
351
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 1, count: 1 });
|
|
352
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
|
|
359
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 1, count: 1 });
|
|
360
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 1, count: 1 });
|
|
353
361
|
|
|
354
362
|
const line = await state.buildNextCheckpointLine({
|
|
355
363
|
base: storage.makeCheckpoint(1n),
|
|
@@ -359,8 +367,8 @@ bucket_definitions:
|
|
|
359
367
|
|
|
360
368
|
line!.advance();
|
|
361
369
|
|
|
362
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 2, count: 2 });
|
|
363
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 2, count: 2 });
|
|
370
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 2, count: 2 });
|
|
371
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 2, count: 2 });
|
|
364
372
|
|
|
365
373
|
const line2 = (await state.buildNextCheckpointLine({
|
|
366
374
|
base: storage.makeCheckpoint(2n),
|
|
@@ -372,24 +380,24 @@ bucket_definitions:
|
|
|
372
380
|
checkpoint_diff: {
|
|
373
381
|
removed_buckets: [],
|
|
374
382
|
updated_buckets: [
|
|
375
|
-
{ bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
376
|
-
{ bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
383
|
+
{ bucket: '2#global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
|
|
384
|
+
{ bucket: '2#global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
|
|
377
385
|
],
|
|
378
386
|
last_op_id: '2',
|
|
379
387
|
write_checkpoint: undefined
|
|
380
388
|
}
|
|
381
389
|
});
|
|
382
390
|
expect(line2.bucketsToFetch).toEqual([
|
|
383
|
-
{ bucket: 'global[1]', priority: 3 },
|
|
384
|
-
{ bucket: 'global[2]', priority: 3 }
|
|
391
|
+
{ bucket: '2#global[1]', priority: 3 },
|
|
392
|
+
{ bucket: '2#global[2]', priority: 3 }
|
|
385
393
|
]);
|
|
386
394
|
});
|
|
387
395
|
|
|
388
396
|
test('interrupt and resume static buckets checkpoint', async () => {
|
|
389
397
|
const storage = new MockBucketChecksumStateStorage();
|
|
390
398
|
// Set intial state
|
|
391
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 3, count: 3 });
|
|
392
|
-
storage.updateTestChecksum({ bucket: 'global[2]', checksum: 3, count: 3 });
|
|
399
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 3, count: 3 });
|
|
400
|
+
storage.updateTestChecksum({ bucket: '2#global[2]', checksum: 3, count: 3 });
|
|
393
401
|
|
|
394
402
|
const state = new BucketChecksumState({
|
|
395
403
|
syncContext,
|
|
@@ -408,8 +416,8 @@ bucket_definitions:
|
|
|
408
416
|
expect(line.checkpointLine).toEqual({
|
|
409
417
|
checkpoint: {
|
|
410
418
|
buckets: [
|
|
411
|
-
{ bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
412
|
-
{ bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
419
|
+
{ bucket: '2#global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
|
|
420
|
+
{ bucket: '2#global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
|
|
413
421
|
],
|
|
414
422
|
last_op_id: '3',
|
|
415
423
|
write_checkpoint: undefined,
|
|
@@ -418,11 +426,11 @@ bucket_definitions:
|
|
|
418
426
|
});
|
|
419
427
|
expect(line.bucketsToFetch).toEqual([
|
|
420
428
|
{
|
|
421
|
-
bucket: 'global[1]',
|
|
429
|
+
bucket: '2#global[1]',
|
|
422
430
|
priority: 3
|
|
423
431
|
},
|
|
424
432
|
{
|
|
425
|
-
bucket: 'global[2]',
|
|
433
|
+
bucket: '2#global[2]',
|
|
426
434
|
priority: 3
|
|
427
435
|
}
|
|
428
436
|
]);
|
|
@@ -430,17 +438,17 @@ bucket_definitions:
|
|
|
430
438
|
// This is the bucket data to be fetched
|
|
431
439
|
expect(line.getFilteredBucketPositions()).toEqual(
|
|
432
440
|
new Map([
|
|
433
|
-
['global[1]', 0n],
|
|
434
|
-
['global[2]', 0n]
|
|
441
|
+
['2#global[1]', 0n],
|
|
442
|
+
['2#global[2]', 0n]
|
|
435
443
|
])
|
|
436
444
|
);
|
|
437
445
|
|
|
438
446
|
// No data changes here.
|
|
439
447
|
// We simulate partial data sent, before a checkpoint is interrupted.
|
|
440
448
|
line.advance();
|
|
441
|
-
line.updateBucketPosition({ bucket: 'global[1]', nextAfter: 3n, hasMore: false });
|
|
442
|
-
line.updateBucketPosition({ bucket: 'global[2]', nextAfter: 1n, hasMore: true });
|
|
443
|
-
storage.updateTestChecksum({ bucket: 'global[1]', checksum: 4, count: 4 });
|
|
449
|
+
line.updateBucketPosition({ bucket: '2#global[1]', nextAfter: 3n, hasMore: false });
|
|
450
|
+
line.updateBucketPosition({ bucket: '2#global[2]', nextAfter: 1n, hasMore: true });
|
|
451
|
+
storage.updateTestChecksum({ bucket: '2#global[1]', checksum: 4, count: 4 });
|
|
444
452
|
|
|
445
453
|
const line2 = (await state.buildNextCheckpointLine({
|
|
446
454
|
base: storage.makeCheckpoint(4n),
|
|
@@ -448,7 +456,7 @@ bucket_definitions:
|
|
|
448
456
|
update: {
|
|
449
457
|
...CHECKPOINT_INVALIDATE_ALL,
|
|
450
458
|
invalidateDataBuckets: false,
|
|
451
|
-
updatedDataBuckets: new Set(['global[1]'])
|
|
459
|
+
updatedDataBuckets: new Set(['2#global[1]'])
|
|
452
460
|
}
|
|
453
461
|
}))!;
|
|
454
462
|
line2.advance();
|
|
@@ -457,7 +465,7 @@ bucket_definitions:
|
|
|
457
465
|
removed_buckets: [],
|
|
458
466
|
updated_buckets: [
|
|
459
467
|
{
|
|
460
|
-
bucket: 'global[1]',
|
|
468
|
+
bucket: '2#global[1]',
|
|
461
469
|
checksum: 4,
|
|
462
470
|
count: 4,
|
|
463
471
|
priority: 3,
|
|
@@ -471,19 +479,19 @@ bucket_definitions:
|
|
|
471
479
|
// This should contain both buckets, even though only one changed.
|
|
472
480
|
expect(line2.bucketsToFetch).toEqual([
|
|
473
481
|
{
|
|
474
|
-
bucket: 'global[1]',
|
|
482
|
+
bucket: '2#global[1]',
|
|
475
483
|
priority: 3
|
|
476
484
|
},
|
|
477
485
|
{
|
|
478
|
-
bucket: 'global[2]',
|
|
486
|
+
bucket: '2#global[2]',
|
|
479
487
|
priority: 3
|
|
480
488
|
}
|
|
481
489
|
]);
|
|
482
490
|
|
|
483
491
|
expect(line2.getFilteredBucketPositions()).toEqual(
|
|
484
492
|
new Map([
|
|
485
|
-
['global[1]', 3n],
|
|
486
|
-
['global[2]', 1n]
|
|
493
|
+
['2#global[1]', 3n],
|
|
494
|
+
['2#global[2]', 1n]
|
|
487
495
|
])
|
|
488
496
|
);
|
|
489
497
|
});
|
|
@@ -491,13 +499,13 @@ bucket_definitions:
|
|
|
491
499
|
test('dynamic buckets with updates', async () => {
|
|
492
500
|
const storage = new MockBucketChecksumStateStorage();
|
|
493
501
|
// Set intial state
|
|
494
|
-
storage.updateTestChecksum({ bucket: 'by_project[1]', checksum: 1, count: 1 });
|
|
495
|
-
storage.updateTestChecksum({ bucket: 'by_project[2]', checksum: 1, count: 1 });
|
|
496
|
-
storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
|
|
502
|
+
storage.updateTestChecksum({ bucket: '3#by_project[1]', checksum: 1, count: 1 });
|
|
503
|
+
storage.updateTestChecksum({ bucket: '3#by_project[2]', checksum: 1, count: 1 });
|
|
504
|
+
storage.updateTestChecksum({ bucket: '3#by_project[3]', checksum: 1, count: 1 });
|
|
497
505
|
|
|
498
506
|
const state = new BucketChecksumState({
|
|
499
507
|
syncContext,
|
|
500
|
-
tokenPayload: { sub: 'u1' },
|
|
508
|
+
tokenPayload: new JwtPayload({ sub: 'u1' }),
|
|
501
509
|
syncRequest,
|
|
502
510
|
syncRules: SYNC_RULES_DYNAMIC,
|
|
503
511
|
bucketStorage: storage
|
|
@@ -515,14 +523,14 @@ bucket_definitions:
|
|
|
515
523
|
checkpoint: {
|
|
516
524
|
buckets: [
|
|
517
525
|
{
|
|
518
|
-
bucket: 'by_project[1]',
|
|
526
|
+
bucket: '3#by_project[1]',
|
|
519
527
|
checksum: 1,
|
|
520
528
|
count: 1,
|
|
521
529
|
priority: 3,
|
|
522
530
|
subscriptions: [{ default: 0 }]
|
|
523
531
|
},
|
|
524
532
|
{
|
|
525
|
-
bucket: 'by_project[2]',
|
|
533
|
+
bucket: '3#by_project[2]',
|
|
526
534
|
checksum: 1,
|
|
527
535
|
count: 1,
|
|
528
536
|
priority: 3,
|
|
@@ -542,11 +550,11 @@ bucket_definitions:
|
|
|
542
550
|
});
|
|
543
551
|
expect(line.bucketsToFetch).toEqual([
|
|
544
552
|
{
|
|
545
|
-
bucket: 'by_project[1]',
|
|
553
|
+
bucket: '3#by_project[1]',
|
|
546
554
|
priority: 3
|
|
547
555
|
},
|
|
548
556
|
{
|
|
549
|
-
bucket: 'by_project[2]',
|
|
557
|
+
bucket: '3#by_project[2]',
|
|
550
558
|
priority: 3
|
|
551
559
|
}
|
|
552
560
|
]);
|
|
@@ -554,14 +562,14 @@ bucket_definitions:
|
|
|
554
562
|
// This is the bucket data to be fetched
|
|
555
563
|
expect(line.getFilteredBucketPositions()).toEqual(
|
|
556
564
|
new Map([
|
|
557
|
-
['by_project[1]', 0n],
|
|
558
|
-
['by_project[2]', 0n]
|
|
565
|
+
['3#by_project[1]', 0n],
|
|
566
|
+
['3#by_project[2]', 0n]
|
|
559
567
|
])
|
|
560
568
|
);
|
|
561
569
|
|
|
562
570
|
line.advance();
|
|
563
|
-
line.updateBucketPosition({ bucket: 'by_project[1]', nextAfter: 1n, hasMore: false });
|
|
564
|
-
line.updateBucketPosition({ bucket: 'by_project[2]', nextAfter: 1n, hasMore: false });
|
|
571
|
+
line.updateBucketPosition({ bucket: '3#by_project[1]', nextAfter: 1n, hasMore: false });
|
|
572
|
+
line.updateBucketPosition({ bucket: '3#by_project[2]', nextAfter: 1n, hasMore: false });
|
|
565
573
|
|
|
566
574
|
// Now we get a new line
|
|
567
575
|
const line2 = (await state.buildNextCheckpointLine({
|
|
@@ -583,7 +591,7 @@ bucket_definitions:
|
|
|
583
591
|
removed_buckets: [],
|
|
584
592
|
updated_buckets: [
|
|
585
593
|
{
|
|
586
|
-
bucket: 'by_project[3]',
|
|
594
|
+
bucket: '3#by_project[3]',
|
|
587
595
|
checksum: 1,
|
|
588
596
|
count: 1,
|
|
589
597
|
priority: 3,
|
|
@@ -594,7 +602,7 @@ bucket_definitions:
|
|
|
594
602
|
write_checkpoint: undefined
|
|
595
603
|
}
|
|
596
604
|
});
|
|
597
|
-
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
|
|
605
|
+
expect(line2.getFilteredBucketPositions()).toEqual(new Map([['3#by_project[3]', 0n]]));
|
|
598
606
|
});
|
|
599
607
|
|
|
600
608
|
describe('streams', () => {
|
|
@@ -615,7 +623,7 @@ config:
|
|
|
615
623
|
|
|
616
624
|
const rules = SqlSyncRules.fromYaml(source, {
|
|
617
625
|
defaultSchema: 'public'
|
|
618
|
-
}).hydrate({ hydrationState: versionedHydrationState(1) });
|
|
626
|
+
}).config.hydrate({ hydrationState: versionedHydrationState(1) });
|
|
619
627
|
|
|
620
628
|
return new BucketChecksumState({
|
|
621
629
|
syncContext,
|
package/test/tsconfig.json
CHANGED
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
{
|
|
2
|
-
"extends": "../../../tsconfig.
|
|
2
|
+
"extends": "../../../tsconfig.tests.json",
|
|
3
3
|
"compilerOptions": {
|
|
4
|
-
"rootDir": "src",
|
|
5
4
|
"baseUrl": "./",
|
|
6
5
|
"outDir": "dist",
|
|
7
|
-
"esModuleInterop": true,
|
|
8
|
-
"skipLibCheck": true,
|
|
9
|
-
"sourceMap": true,
|
|
10
6
|
"paths": {
|
|
11
7
|
"@/*": ["../src/*"]
|
|
12
|
-
}
|
|
8
|
+
},
|
|
9
|
+
"rootDir": "src"
|
|
13
10
|
},
|
|
14
11
|
"include": ["src"],
|
|
15
12
|
"references": [
|