@powersync/service-core 1.19.2 → 1.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/dist/api/diagnostics.js +11 -4
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +13 -2
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/config-command.js +2 -2
- package/dist/entry/commands/config-command.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +84 -0
- package/dist/routes/endpoints/admin.d.ts +168 -0
- package/dist/routes/endpoints/admin.js +34 -20
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +6 -9
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +6 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +21 -8
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +48 -15
- package/dist/storage/BucketStorageFactory.js +70 -1
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +5 -2
- package/dist/storage/ChecksumCache.js +8 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +33 -3
- package/dist/storage/PersistedSyncRulesContent.js +80 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/SourceTable.d.ts +7 -2
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +53 -0
- package/dist/storage/StorageVersionConfig.js +53 -0
- package/dist/storage/StorageVersionConfig.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +14 -4
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +8 -4
- package/dist/sync/BucketChecksumState.js +97 -52
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -0
- package/dist/sync/util.js +10 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/collectors/config-collector.js +13 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +4 -4
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +2 -2
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +3 -3
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -1
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/env.d.ts +1 -0
- package/dist/util/env.js +5 -0
- package/dist/util/env.js.map +1 -1
- package/package.json +6 -6
- package/src/api/diagnostics.ts +12 -4
- package/src/entry/commands/compact-action.ts +15 -2
- package/src/entry/commands/config-command.ts +3 -3
- package/src/replication/AbstractReplicator.ts +3 -5
- package/src/routes/endpoints/admin.ts +43 -25
- package/src/routes/endpoints/sync-rules.ts +14 -13
- package/src/routes/endpoints/sync-stream.ts +6 -1
- package/src/storage/BucketStorageBatch.ts +23 -9
- package/src/storage/BucketStorageFactory.ts +116 -19
- package/src/storage/ChecksumCache.ts +14 -6
- package/src/storage/PersistedSyncRulesContent.ts +119 -4
- package/src/storage/SourceTable.ts +7 -1
- package/src/storage/StorageVersionConfig.ts +78 -0
- package/src/storage/SyncRulesBucketStorage.ts +20 -4
- package/src/storage/storage-index.ts +1 -0
- package/src/sync/BucketChecksumState.ts +147 -65
- package/src/sync/sync.ts +9 -3
- package/src/sync/util.ts +10 -0
- package/src/util/config/collectors/config-collector.ts +16 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +5 -5
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +4 -4
- package/src/util/config/types.ts +1 -2
- package/src/util/env.ts +5 -0
- package/test/src/checksum_cache.test.ts +102 -57
- package/test/src/config.test.ts +115 -0
- package/test/src/routes/admin.test.ts +48 -0
- package/test/src/routes/mocks.ts +22 -1
- package/test/src/routes/stream.test.ts +3 -2
- package/test/src/sync/BucketChecksumState.test.ts +332 -93
- package/test/src/utils.ts +9 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
|
|
2
2
|
import { addChecksums, BucketChecksum, InternalOpId, PartialChecksum } from '@/util/util-index.js';
|
|
3
|
+
import { BucketDataSource } from '@powersync/service-sync-rules';
|
|
3
4
|
import * as crypto from 'node:crypto';
|
|
4
5
|
import { describe, expect, it } from 'vitest';
|
|
6
|
+
import { removeSource } from './utils.js';
|
|
5
7
|
|
|
6
8
|
/**
|
|
7
9
|
* Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.
|
|
@@ -67,6 +69,12 @@ describe('checksum cache', function () {
|
|
|
67
69
|
return new ChecksumCache({ fetchChecksums: fetch });
|
|
68
70
|
};
|
|
69
71
|
|
|
72
|
+
const DUMMY_SOURCE: BucketDataSource = null as any;
|
|
73
|
+
|
|
74
|
+
function removeLookupSources(lookups: FetchPartialBucketChecksum[]) {
|
|
75
|
+
return lookups.map((b) => removeSource(b));
|
|
76
|
+
}
|
|
77
|
+
|
|
70
78
|
it('should handle a sequential lookups (a)', async function () {
|
|
71
79
|
let lookups: FetchPartialBucketChecksum[][] = [];
|
|
72
80
|
const cache = factory(async (batch) => {
|
|
@@ -74,13 +82,13 @@ describe('checksum cache', function () {
|
|
|
74
82
|
return fetchTestChecksums(batch);
|
|
75
83
|
});
|
|
76
84
|
|
|
77
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
85
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
78
86
|
|
|
79
|
-
expect(await cache.getChecksums(1234n, ['test'])).toEqual([TEST_1234]);
|
|
87
|
+
expect(await cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_1234]);
|
|
80
88
|
|
|
81
|
-
expect(await cache.getChecksums(123n, ['test2'])).toEqual([TEST2_123]);
|
|
89
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test2', source: DUMMY_SOURCE }])).toEqual([TEST2_123]);
|
|
82
90
|
|
|
83
|
-
expect(lookups).
|
|
91
|
+
expect(lookups.map(removeLookupSources)).toMatchObject([
|
|
84
92
|
[{ bucket: 'test', end: 123n }],
|
|
85
93
|
// This should use the previous lookup
|
|
86
94
|
[{ bucket: 'test', start: 123n, end: 1234n }],
|
|
@@ -96,13 +104,13 @@ describe('checksum cache', function () {
|
|
|
96
104
|
return fetchTestChecksums(batch);
|
|
97
105
|
});
|
|
98
106
|
|
|
99
|
-
expect(await cache.getChecksums(123n, ['test2'])).toEqual([TEST2_123]);
|
|
107
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test2', source: DUMMY_SOURCE }])).toEqual([TEST2_123]);
|
|
100
108
|
|
|
101
|
-
expect(await cache.getChecksums(1234n, ['test'])).toEqual([TEST_1234]);
|
|
109
|
+
expect(await cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_1234]);
|
|
102
110
|
|
|
103
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
111
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
104
112
|
|
|
105
|
-
expect(lookups).toEqual([
|
|
113
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
106
114
|
// With this order, there is no option for a partial lookup
|
|
107
115
|
[{ bucket: 'test2', end: 123n }],
|
|
108
116
|
[{ bucket: 'test', end: 1234n }],
|
|
@@ -117,16 +125,16 @@ describe('checksum cache', function () {
|
|
|
117
125
|
return fetchTestChecksums(batch);
|
|
118
126
|
});
|
|
119
127
|
|
|
120
|
-
const p1 = cache.getChecksums(123n, ['test']);
|
|
121
|
-
const p2 = cache.getChecksums(1234n, ['test']);
|
|
122
|
-
const p3 = cache.getChecksums(123n, ['test2']);
|
|
128
|
+
const p1 = cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
129
|
+
const p2 = cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
130
|
+
const p3 = cache.getChecksums(123n, [{ bucket: 'test2', source: DUMMY_SOURCE }]);
|
|
123
131
|
|
|
124
132
|
expect(await p1).toEqual([TEST_123]);
|
|
125
133
|
expect(await p2).toEqual([TEST_1234]);
|
|
126
134
|
expect(await p3).toEqual([TEST2_123]);
|
|
127
135
|
|
|
128
136
|
// Concurrent requests, so we can't do a partial lookup for 123 -> 1234
|
|
129
|
-
expect(lookups).toEqual([
|
|
137
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
130
138
|
[{ bucket: 'test', end: 123n }],
|
|
131
139
|
[{ bucket: 'test', end: 1234n }],
|
|
132
140
|
[{ bucket: 'test2', end: 123n }]
|
|
@@ -140,15 +148,15 @@ describe('checksum cache', function () {
|
|
|
140
148
|
return fetchTestChecksums(batch);
|
|
141
149
|
});
|
|
142
150
|
|
|
143
|
-
const p1 = cache.getChecksums(123n, ['test']);
|
|
144
|
-
const p2 = cache.getChecksums(123n, ['test']);
|
|
151
|
+
const p1 = cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
152
|
+
const p2 = cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
145
153
|
|
|
146
154
|
expect(await p1).toEqual([TEST_123]);
|
|
147
155
|
|
|
148
156
|
expect(await p2).toEqual([TEST_123]);
|
|
149
157
|
|
|
150
158
|
// The lookup should be deduplicated, even though it's in progress
|
|
151
|
-
expect(lookups).toEqual([[{ bucket: 'test', end: 123n }]]);
|
|
159
|
+
expect(lookups.map(removeLookupSources)).toEqual([[{ bucket: 'test', end: 123n }]]);
|
|
152
160
|
});
|
|
153
161
|
|
|
154
162
|
it('should handle serial + concurrent lookups', async function () {
|
|
@@ -158,15 +166,15 @@ describe('checksum cache', function () {
|
|
|
158
166
|
return fetchTestChecksums(batch);
|
|
159
167
|
});
|
|
160
168
|
|
|
161
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
169
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
162
170
|
|
|
163
|
-
const p2 = cache.getChecksums(1234n, ['test']);
|
|
164
|
-
const p3 = cache.getChecksums(1234n, ['test']);
|
|
171
|
+
const p2 = cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
172
|
+
const p3 = cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
165
173
|
|
|
166
174
|
expect(await p2).toEqual([TEST_1234]);
|
|
167
175
|
expect(await p3).toEqual([TEST_1234]);
|
|
168
176
|
|
|
169
|
-
expect(lookups).toEqual([
|
|
177
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
170
178
|
[{ bucket: 'test', end: 123n }],
|
|
171
179
|
// This lookup is deduplicated
|
|
172
180
|
[{ bucket: 'test', start: 123n, end: 1234n }]
|
|
@@ -180,9 +188,14 @@ describe('checksum cache', function () {
|
|
|
180
188
|
return fetchTestChecksums(batch);
|
|
181
189
|
});
|
|
182
190
|
|
|
183
|
-
expect(
|
|
191
|
+
expect(
|
|
192
|
+
await cache.getChecksums(123n, [
|
|
193
|
+
{ bucket: 'test', source: DUMMY_SOURCE },
|
|
194
|
+
{ bucket: 'test2', source: DUMMY_SOURCE }
|
|
195
|
+
])
|
|
196
|
+
).toEqual([TEST_123, TEST2_123]);
|
|
184
197
|
|
|
185
|
-
expect(lookups).toEqual([
|
|
198
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
186
199
|
[
|
|
187
200
|
// Both lookups in the same request
|
|
188
201
|
{ bucket: 'test', end: 123n },
|
|
@@ -198,10 +211,15 @@ describe('checksum cache', function () {
|
|
|
198
211
|
return fetchTestChecksums(batch);
|
|
199
212
|
});
|
|
200
213
|
|
|
201
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
202
|
-
expect(
|
|
214
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
215
|
+
expect(
|
|
216
|
+
await cache.getChecksums(123n, [
|
|
217
|
+
{ bucket: 'test', source: DUMMY_SOURCE },
|
|
218
|
+
{ bucket: 'test2', source: DUMMY_SOURCE }
|
|
219
|
+
])
|
|
220
|
+
).toEqual([TEST_123, TEST2_123]);
|
|
203
221
|
|
|
204
|
-
expect(lookups).toEqual([
|
|
222
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
205
223
|
// Request 1
|
|
206
224
|
[{ bucket: 'test', end: 123n }],
|
|
207
225
|
// Request 2
|
|
@@ -216,13 +234,19 @@ describe('checksum cache', function () {
|
|
|
216
234
|
return fetchTestChecksums(batch);
|
|
217
235
|
});
|
|
218
236
|
|
|
219
|
-
const a = cache.getChecksums(123n, [
|
|
220
|
-
|
|
237
|
+
const a = cache.getChecksums(123n, [
|
|
238
|
+
{ bucket: 'test', source: DUMMY_SOURCE },
|
|
239
|
+
{ bucket: 'test2', source: DUMMY_SOURCE }
|
|
240
|
+
]);
|
|
241
|
+
const b = cache.getChecksums(123n, [
|
|
242
|
+
{ bucket: 'test2', source: DUMMY_SOURCE },
|
|
243
|
+
{ bucket: 'test3', source: DUMMY_SOURCE }
|
|
244
|
+
]);
|
|
221
245
|
|
|
222
246
|
expect(await a).toEqual([TEST_123, TEST2_123]);
|
|
223
247
|
expect(await b).toEqual([TEST2_123, TEST3_123]);
|
|
224
248
|
|
|
225
|
-
expect(lookups).toEqual([
|
|
249
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
226
250
|
// Request A
|
|
227
251
|
[
|
|
228
252
|
{ bucket: 'test', end: 123n },
|
|
@@ -240,9 +264,9 @@ describe('checksum cache', function () {
|
|
|
240
264
|
return fetchTestChecksums(batch);
|
|
241
265
|
});
|
|
242
266
|
|
|
243
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
267
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
244
268
|
|
|
245
|
-
expect(await cache.getChecksums(125n, ['test'])).toEqual([
|
|
269
|
+
expect(await cache.getChecksums(125n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
246
270
|
{
|
|
247
271
|
bucket: 'test',
|
|
248
272
|
checksum: -1865121912,
|
|
@@ -250,14 +274,14 @@ describe('checksum cache', function () {
|
|
|
250
274
|
}
|
|
251
275
|
]);
|
|
252
276
|
|
|
253
|
-
expect(await cache.getChecksums(124n, ['test'])).toEqual([
|
|
277
|
+
expect(await cache.getChecksums(124n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
254
278
|
{
|
|
255
279
|
bucket: 'test',
|
|
256
280
|
checksum: 1887460431,
|
|
257
281
|
count: 124
|
|
258
282
|
}
|
|
259
283
|
]);
|
|
260
|
-
expect(lookups).toEqual([
|
|
284
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
261
285
|
[{ bucket: 'test', end: 123n }],
|
|
262
286
|
[{ bucket: 'test', start: 123n, end: 125n }],
|
|
263
287
|
[{ bucket: 'test', start: 123n, end: 124n }]
|
|
@@ -275,19 +299,31 @@ describe('checksum cache', function () {
|
|
|
275
299
|
return fetchTestChecksums(batch);
|
|
276
300
|
});
|
|
277
301
|
|
|
278
|
-
const a = cache.getChecksums(123n, [
|
|
279
|
-
|
|
302
|
+
const a = cache.getChecksums(123n, [
|
|
303
|
+
{ bucket: 'test', source: DUMMY_SOURCE },
|
|
304
|
+
{ bucket: 'test2', source: DUMMY_SOURCE }
|
|
305
|
+
]);
|
|
306
|
+
const b = cache.getChecksums(123n, [
|
|
307
|
+
{ bucket: 'test2', source: DUMMY_SOURCE },
|
|
308
|
+
{ bucket: 'test3', source: DUMMY_SOURCE }
|
|
309
|
+
]);
|
|
280
310
|
|
|
281
311
|
await expect(a).rejects.toEqual(TEST_ERROR);
|
|
282
312
|
await expect(b).rejects.toEqual(TEST_ERROR);
|
|
283
313
|
|
|
284
|
-
const a2 = cache.getChecksums(123n, [
|
|
285
|
-
|
|
314
|
+
const a2 = cache.getChecksums(123n, [
|
|
315
|
+
{ bucket: 'test', source: DUMMY_SOURCE },
|
|
316
|
+
{ bucket: 'test2', source: DUMMY_SOURCE }
|
|
317
|
+
]);
|
|
318
|
+
const b2 = cache.getChecksums(123n, [
|
|
319
|
+
{ bucket: 'test2', source: DUMMY_SOURCE },
|
|
320
|
+
{ bucket: 'test3', source: DUMMY_SOURCE }
|
|
321
|
+
]);
|
|
286
322
|
|
|
287
323
|
expect(await a2).toEqual([TEST_123, TEST2_123]);
|
|
288
324
|
expect(await b2).toEqual([TEST2_123, TEST3_123]);
|
|
289
325
|
|
|
290
|
-
expect(lookups).toEqual([
|
|
326
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
291
327
|
// Request A (fails)
|
|
292
328
|
[
|
|
293
329
|
{ bucket: 'test', end: 123n },
|
|
@@ -311,11 +347,15 @@ describe('checksum cache', function () {
|
|
|
311
347
|
return fetchTestChecksums(batch.filter((b) => b.bucket != 'test'));
|
|
312
348
|
});
|
|
313
349
|
|
|
314
|
-
expect(await cache.getChecksums(123n, [
|
|
315
|
-
|
|
316
|
-
{ bucket: 'test', checksum: 0, count: 0 },
|
|
317
|
-
TEST2_123
|
|
350
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
351
|
+
{ bucket: 'test', checksum: 0, count: 0 }
|
|
318
352
|
]);
|
|
353
|
+
expect(
|
|
354
|
+
await cache.getChecksums(123n, [
|
|
355
|
+
{ bucket: 'test', source: DUMMY_SOURCE },
|
|
356
|
+
{ bucket: 'test2', source: DUMMY_SOURCE }
|
|
357
|
+
])
|
|
358
|
+
).toEqual([{ bucket: 'test', checksum: 0, count: 0 }, TEST2_123]);
|
|
319
359
|
});
|
|
320
360
|
|
|
321
361
|
it('should handle missing checksums (b)', async function () {
|
|
@@ -325,8 +365,10 @@ describe('checksum cache', function () {
|
|
|
325
365
|
return fetchTestChecksums(batch.filter((b) => b.bucket != 'test' || b.end != 123n));
|
|
326
366
|
});
|
|
327
367
|
|
|
328
|
-
expect(await cache.getChecksums(123n, [
|
|
329
|
-
|
|
368
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
369
|
+
{ bucket: 'test', checksum: 0, count: 0 }
|
|
370
|
+
]);
|
|
371
|
+
expect(await cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
330
372
|
{
|
|
331
373
|
bucket: 'test',
|
|
332
374
|
checksum: 1597020602,
|
|
@@ -334,7 +376,10 @@ describe('checksum cache', function () {
|
|
|
334
376
|
}
|
|
335
377
|
]);
|
|
336
378
|
|
|
337
|
-
expect(lookups).toEqual([
|
|
379
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
380
|
+
[{ bucket: 'test', end: 123n }],
|
|
381
|
+
[{ bucket: 'test', start: 123n, end: 1234n }]
|
|
382
|
+
]);
|
|
338
383
|
});
|
|
339
384
|
|
|
340
385
|
it('should use maxSize', async function () {
|
|
@@ -347,8 +392,8 @@ describe('checksum cache', function () {
|
|
|
347
392
|
maxSize: 2
|
|
348
393
|
});
|
|
349
394
|
|
|
350
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
351
|
-
expect(await cache.getChecksums(124n, ['test'])).toEqual([
|
|
395
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
396
|
+
expect(await cache.getChecksums(124n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
352
397
|
{
|
|
353
398
|
bucket: 'test',
|
|
354
399
|
checksum: 1887460431,
|
|
@@ -356,30 +401,30 @@ describe('checksum cache', function () {
|
|
|
356
401
|
}
|
|
357
402
|
]);
|
|
358
403
|
|
|
359
|
-
expect(await cache.getChecksums(125n, ['test'])).toEqual([
|
|
404
|
+
expect(await cache.getChecksums(125n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
360
405
|
{
|
|
361
406
|
bucket: 'test',
|
|
362
407
|
checksum: -1865121912,
|
|
363
408
|
count: 125
|
|
364
409
|
}
|
|
365
410
|
]);
|
|
366
|
-
expect(await cache.getChecksums(126n, ['test'])).toEqual([
|
|
411
|
+
expect(await cache.getChecksums(126n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
367
412
|
{
|
|
368
413
|
bucket: 'test',
|
|
369
414
|
checksum: -1720007310,
|
|
370
415
|
count: 126
|
|
371
416
|
}
|
|
372
417
|
]);
|
|
373
|
-
expect(await cache.getChecksums(124n, ['test'])).toEqual([
|
|
418
|
+
expect(await cache.getChecksums(124n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([
|
|
374
419
|
{
|
|
375
420
|
bucket: 'test',
|
|
376
421
|
checksum: 1887460431,
|
|
377
422
|
count: 124
|
|
378
423
|
}
|
|
379
424
|
]);
|
|
380
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
425
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
381
426
|
|
|
382
|
-
expect(lookups).toEqual([
|
|
427
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
383
428
|
[{ bucket: 'test', end: 123n }],
|
|
384
429
|
[{ bucket: 'test', start: 123n, end: 124n }],
|
|
385
430
|
[{ bucket: 'test', start: 124n, end: 125n }],
|
|
@@ -400,10 +445,10 @@ describe('checksum cache', function () {
|
|
|
400
445
|
maxSize: 2
|
|
401
446
|
});
|
|
402
447
|
|
|
403
|
-
const p3 = cache.getChecksums(123n, ['test3']);
|
|
404
|
-
const p4 = cache.getChecksums(123n, ['test4']);
|
|
405
|
-
const p1 = cache.getChecksums(123n, ['test']);
|
|
406
|
-
const p2 = cache.getChecksums(123n, ['test2']);
|
|
448
|
+
const p3 = cache.getChecksums(123n, [{ bucket: 'test3', source: DUMMY_SOURCE }]);
|
|
449
|
+
const p4 = cache.getChecksums(123n, [{ bucket: 'test4', source: DUMMY_SOURCE }]);
|
|
450
|
+
const p1 = cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }]);
|
|
451
|
+
const p2 = cache.getChecksums(123n, [{ bucket: 'test2', source: DUMMY_SOURCE }]);
|
|
407
452
|
|
|
408
453
|
expect(await p1).toEqual([TEST_123]);
|
|
409
454
|
expect(await p2).toEqual([TEST2_123]);
|
|
@@ -417,7 +462,7 @@ describe('checksum cache', function () {
|
|
|
417
462
|
]);
|
|
418
463
|
|
|
419
464
|
// The lookup should be deduplicated, even though it's in progress
|
|
420
|
-
expect(lookups).toEqual([
|
|
465
|
+
expect(lookups.map(removeLookupSources)).toEqual([
|
|
421
466
|
[{ bucket: 'test3', end: 123n }],
|
|
422
467
|
[{ bucket: 'test4', end: 123n }],
|
|
423
468
|
[{ bucket: 'test', end: 123n }],
|
|
@@ -434,7 +479,7 @@ describe('checksum cache', function () {
|
|
|
434
479
|
return fetchTestChecksums(batch);
|
|
435
480
|
});
|
|
436
481
|
|
|
437
|
-
expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
|
|
438
|
-
expect(await cache.getChecksums(1234n, ['test'])).toEqual([TEST_1234]);
|
|
482
|
+
expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
|
|
483
|
+
expect(await cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_1234]);
|
|
439
484
|
});
|
|
440
485
|
});
|
package/test/src/config.test.ts
CHANGED
|
@@ -69,6 +69,7 @@ describe('Config', () => {
|
|
|
69
69
|
|
|
70
70
|
expect(config.api_parameters.max_buckets_per_connection).toBe(1);
|
|
71
71
|
});
|
|
72
|
+
|
|
72
73
|
it('should throw YAML validation error for invalid base64 config', {}, async () => {
|
|
73
74
|
const yamlConfig = /* yaml */ `
|
|
74
75
|
# PowerSync config
|
|
@@ -86,4 +87,118 @@ describe('Config', () => {
|
|
|
86
87
|
})
|
|
87
88
|
).rejects.toThrow(/YAML Error:[\s\S]*Attempting to substitute environment variable INVALID_VAR/);
|
|
88
89
|
});
|
|
90
|
+
|
|
91
|
+
it('should resolve inline sync config', async () => {
|
|
92
|
+
const yamlConfig = /* yaml */ `
|
|
93
|
+
# PowerSync config
|
|
94
|
+
replication:
|
|
95
|
+
connections: []
|
|
96
|
+
storage:
|
|
97
|
+
type: mongodb
|
|
98
|
+
sync_config:
|
|
99
|
+
content: |
|
|
100
|
+
config:
|
|
101
|
+
edition: 2
|
|
102
|
+
streams:
|
|
103
|
+
a:
|
|
104
|
+
query: SELECT * FROM users
|
|
105
|
+
`;
|
|
106
|
+
|
|
107
|
+
const collector = new CompoundConfigCollector();
|
|
108
|
+
|
|
109
|
+
const result = await collector.collectConfig({
|
|
110
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
expect(result.sync_rules).toEqual({
|
|
114
|
+
present: true,
|
|
115
|
+
exit_on_error: true,
|
|
116
|
+
content: expect.stringContaining('edition: 2')
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('should still resolve inline sync rules', async () => {
|
|
121
|
+
const yamlConfig = /* yaml */ `
|
|
122
|
+
# PowerSync config
|
|
123
|
+
replication:
|
|
124
|
+
connections: []
|
|
125
|
+
storage:
|
|
126
|
+
type: mongodb
|
|
127
|
+
sync_rules:
|
|
128
|
+
content: |
|
|
129
|
+
config:
|
|
130
|
+
edition: 2
|
|
131
|
+
streams:
|
|
132
|
+
a:
|
|
133
|
+
query: SELECT * FROM users
|
|
134
|
+
`;
|
|
135
|
+
|
|
136
|
+
const collector = new CompoundConfigCollector();
|
|
137
|
+
|
|
138
|
+
const result = await collector.collectConfig({
|
|
139
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
expect(result.sync_rules).toEqual({
|
|
143
|
+
present: true,
|
|
144
|
+
exit_on_error: true,
|
|
145
|
+
content: expect.stringContaining('edition: 2')
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('should resolve base64 sync config', async () => {
|
|
150
|
+
const yamlConfig = /* yaml */ `
|
|
151
|
+
# PowerSync config
|
|
152
|
+
replication:
|
|
153
|
+
connections: []
|
|
154
|
+
storage:
|
|
155
|
+
type: mongodb
|
|
156
|
+
`;
|
|
157
|
+
const yamlSyncConfig = /* yaml */ `
|
|
158
|
+
config:
|
|
159
|
+
edition: 2
|
|
160
|
+
streams:
|
|
161
|
+
a:
|
|
162
|
+
query: SELECT * FROM users
|
|
163
|
+
`;
|
|
164
|
+
|
|
165
|
+
const collector = new CompoundConfigCollector();
|
|
166
|
+
|
|
167
|
+
const result = await collector.collectConfig({
|
|
168
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64'),
|
|
169
|
+
sync_config_base64: Buffer.from(yamlSyncConfig, 'utf-8').toString('base64')
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
expect(result.sync_rules).toEqual({
|
|
173
|
+
present: true,
|
|
174
|
+
exit_on_error: true,
|
|
175
|
+
content: expect.stringContaining('edition: 2')
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('should not allow both sync_config and sync_rules', async () => {
|
|
180
|
+
const yamlConfig = /* yaml */ `
|
|
181
|
+
# PowerSync config
|
|
182
|
+
replication:
|
|
183
|
+
connections: []
|
|
184
|
+
storage:
|
|
185
|
+
type: mongodb
|
|
186
|
+
sync_config:
|
|
187
|
+
content: |
|
|
188
|
+
config:
|
|
189
|
+
edition: 2
|
|
190
|
+
sync_rules:
|
|
191
|
+
content: |
|
|
192
|
+
config:
|
|
193
|
+
edition: 2
|
|
194
|
+
`;
|
|
195
|
+
|
|
196
|
+
const collector = new CompoundConfigCollector();
|
|
197
|
+
|
|
198
|
+
await expect(
|
|
199
|
+
collector.collectConfig({
|
|
200
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
201
|
+
})
|
|
202
|
+
).rejects.toThrow(/Both `sync_config` and `sync_rules` are present/);
|
|
203
|
+
});
|
|
89
204
|
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { BasicRouterRequest, Context, JwtPayload } from '@/index.js';
|
|
2
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import { describe, expect, it } from 'vitest';
|
|
4
|
+
import { validate } from '../../../src/routes/endpoints/admin.js';
|
|
5
|
+
import { mockServiceContext } from './mocks.js';
|
|
6
|
+
|
|
7
|
+
describe('admin routes', () => {
|
|
8
|
+
describe('validate', () => {
|
|
9
|
+
it('reports errors with source location', async () => {
|
|
10
|
+
const context: Context = {
|
|
11
|
+
logger: logger,
|
|
12
|
+
service_context: mockServiceContext(null),
|
|
13
|
+
token_payload: new JwtPayload({
|
|
14
|
+
sub: '',
|
|
15
|
+
exp: 0,
|
|
16
|
+
iat: 0
|
|
17
|
+
})
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const request: BasicRouterRequest = {
|
|
21
|
+
headers: {},
|
|
22
|
+
hostname: '',
|
|
23
|
+
protocol: 'http'
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const response = await validate.handler({
|
|
27
|
+
context,
|
|
28
|
+
params: {
|
|
29
|
+
sync_rules: `
|
|
30
|
+
bucket_definitions:
|
|
31
|
+
missing_table:
|
|
32
|
+
data:
|
|
33
|
+
- SELECT * FROM missing_table
|
|
34
|
+
`
|
|
35
|
+
},
|
|
36
|
+
request
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
expect(response.errors).toEqual([
|
|
40
|
+
expect.objectContaining({
|
|
41
|
+
level: 'warning',
|
|
42
|
+
location: { start_offset: 70, end_offset: 83 },
|
|
43
|
+
message: 'Table public.missing_table not found'
|
|
44
|
+
})
|
|
45
|
+
]);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
});
|
package/test/src/routes/mocks.ts
CHANGED
|
@@ -41,8 +41,29 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
|
|
|
41
41
|
return {
|
|
42
42
|
getParseSyncRulesOptions() {
|
|
43
43
|
return { defaultSchema: 'public' };
|
|
44
|
+
},
|
|
45
|
+
async getSourceConfig() {
|
|
46
|
+
return {
|
|
47
|
+
tag: 'test_tag',
|
|
48
|
+
id: 'test_id',
|
|
49
|
+
type: 'test_type'
|
|
50
|
+
};
|
|
51
|
+
},
|
|
52
|
+
async getConnectionSchema() {
|
|
53
|
+
return [];
|
|
54
|
+
},
|
|
55
|
+
async getConnectionStatus() {
|
|
56
|
+
return {
|
|
57
|
+
id: 'test_id',
|
|
58
|
+
uri: 'http://example.org/',
|
|
59
|
+
connected: true,
|
|
60
|
+
errors: []
|
|
61
|
+
};
|
|
62
|
+
},
|
|
63
|
+
async getDebugTablesInfo() {
|
|
64
|
+
return [];
|
|
44
65
|
}
|
|
45
|
-
}
|
|
66
|
+
} satisfies Partial<RouteAPI> as unknown as RouteAPI;
|
|
46
67
|
},
|
|
47
68
|
addStopHandler() {
|
|
48
69
|
return () => {};
|
|
@@ -8,6 +8,7 @@ import winston from 'winston';
|
|
|
8
8
|
import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
|
|
9
9
|
import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
|
|
10
10
|
import { mockServiceContext } from './mocks.js';
|
|
11
|
+
import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
|
|
11
12
|
|
|
12
13
|
describe('Stream Route', () => {
|
|
13
14
|
describe('compressed stream', () => {
|
|
@@ -45,7 +46,7 @@ describe('Stream Route', () => {
|
|
|
45
46
|
|
|
46
47
|
const storage = {
|
|
47
48
|
getParsedSyncRules() {
|
|
48
|
-
return new SqlSyncRules('bucket_definitions: {}').hydrate();
|
|
49
|
+
return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
|
|
49
50
|
},
|
|
50
51
|
watchCheckpointChanges: async function* (options) {
|
|
51
52
|
throw new Error('Simulated storage error');
|
|
@@ -83,7 +84,7 @@ describe('Stream Route', () => {
|
|
|
83
84
|
it('logs the application metadata', async () => {
|
|
84
85
|
const storage = {
|
|
85
86
|
getParsedSyncRules() {
|
|
86
|
-
return new SqlSyncRules('bucket_definitions: {}').hydrate();
|
|
87
|
+
return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
|
|
87
88
|
},
|
|
88
89
|
watchCheckpointChanges: async function* (options) {
|
|
89
90
|
throw new Error('Simulated storage error');
|