@powersync/service-core 1.27.2 → 1.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/LICENSE +201 -67
- package/dist/api/RouteAPI.d.ts +6 -0
- package/dist/api/api-metrics.js +4 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +5 -0
- package/dist/api/diagnostics.js +17 -2
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/metrics/connection-metrics.d.ts +28 -0
- package/dist/metrics/connection-metrics.js +55 -0
- package/dist/metrics/connection-metrics.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +1 -0
- package/dist/metrics/metrics-index.js +1 -0
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +7 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/modules/loader.js +8 -2
- package/dist/modules/loader.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -0
- package/dist/replication/AbstractReplicationJob.js +17 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +8 -18
- package/dist/replication/AbstractReplicator.js +63 -64
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.js +0 -6
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.js +5 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.js +24 -22
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +19 -27
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +17 -40
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +25 -23
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.d.ts +1 -1
- package/dist/routes/hooks.js +2 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/sync-connection.d.ts +7 -0
- package/dist/routes/sync-connection.js +34 -0
- package/dist/routes/sync-connection.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +30 -22
- package/dist/storage/BucketStorageFactory.js +14 -14
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.js +2 -2
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +23 -2
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-metrics.js +2 -1
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/streams/BroadcastIterable.js +8 -7
- package/dist/streams/BroadcastIterable.js.map +1 -1
- package/dist/streams/LastValueSink.d.ts +2 -2
- package/dist/streams/LastValueSink.js +8 -13
- package/dist/streams/LastValueSink.js.map +1 -1
- package/dist/streams/merge.d.ts +15 -31
- package/dist/streams/merge.js +136 -152
- package/dist/streams/merge.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +3 -2
- package/dist/sync/BucketChecksumState.js +12 -5
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +48 -38
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -2
- package/dist/sync/util.js +2 -5
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +7 -0
- package/dist/system/ServiceContext.js +3 -3
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/system/SyncConfigEngine.d.ts +135 -0
- package/dist/system/SyncConfigEngine.js +186 -0
- package/dist/system/SyncConfigEngine.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -0
- package/dist/system/system-index.js +1 -0
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/types.d.ts +4 -0
- package/package.json +8 -9
- package/src/api/RouteAPI.ts +6 -0
- package/src/api/api-metrics.ts +5 -0
- package/src/api/diagnostics.ts +22 -2
- package/src/metrics/connection-metrics.ts +75 -0
- package/src/metrics/metrics-index.ts +1 -0
- package/src/metrics/metrics-interfaces.ts +8 -1
- package/src/modules/loader.ts +16 -2
- package/src/replication/AbstractReplicationJob.ts +17 -1
- package/src/replication/AbstractReplicator.ts +83 -81
- package/src/replication/ReplicationModule.ts +0 -5
- package/src/routes/configure-fastify.ts +7 -1
- package/src/routes/endpoints/admin.ts +27 -20
- package/src/routes/endpoints/socket-route.ts +26 -38
- package/src/routes/endpoints/sync-rules.ts +22 -46
- package/src/routes/endpoints/sync-stream.ts +31 -27
- package/src/routes/hooks.ts +5 -1
- package/src/routes/sync-connection.ts +39 -0
- package/src/storage/BucketStorageFactory.ts +45 -29
- package/src/storage/PersistedSyncConfigContent.ts +2 -2
- package/src/storage/SyncRulesBucketStorage.ts +23 -2
- package/src/storage/storage-metrics.ts +2 -1
- package/src/streams/BroadcastIterable.ts +8 -7
- package/src/streams/LastValueSink.ts +10 -16
- package/src/streams/merge.ts +149 -169
- package/src/sync/BucketChecksumState.ts +17 -5
- package/src/sync/sync.ts +71 -57
- package/src/sync/util.ts +2 -7
- package/src/system/ServiceContext.ts +8 -3
- package/src/system/SyncConfigEngine.ts +247 -0
- package/src/system/system-index.ts +1 -0
- package/src/util/config/types.ts +4 -0
- package/test/src/AbstractReplicator.test.ts +375 -45
- package/test/src/PersistedSyncConfigContent.test.ts +5 -5
- package/test/src/SyncConfigEngine.test.ts +208 -0
- package/test/src/broadcast_iterable.test.ts +59 -31
- package/test/src/config.test.ts +65 -3
- package/test/src/connection-metrics-export.test.ts +76 -0
- package/test/src/connection-metrics.test.ts +184 -0
- package/test/src/diagnostics.test.ts +3 -3
- package/test/src/last_value_sink.test.ts +34 -0
- package/test/src/merge_iterable.test.ts +304 -55
- package/test/src/module-loader.test.ts +14 -1
- package/test/src/recording-metrics.ts +36 -0
- package/test/src/routes/admin.test.ts +2 -2
- package/test/src/routes/concurrency-limit.test.ts +108 -0
- package/test/src/routes/connection-metrics.test.ts +269 -0
- package/test/src/routes/invalid-config.test.ts +184 -0
- package/test/src/routes/mocks.ts +26 -13
- package/test/src/routes/stream-disconnect.integration.test.ts +129 -0
- package/test/src/routes/stream.test.ts +200 -107
- package/test/src/routes/sync-rules.test.ts +19 -1
- package/test/src/source_table_config.test.ts +7 -7
- package/test/src/sync/BucketChecksumState.test.ts +10 -10
- package/test/src/sync/stream_response.test.ts +389 -0
- package/test/src/sync/util.test.ts +14 -3
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,24 +1,48 @@
|
|
|
1
1
|
import type { RouteAPI } from '@/api/RouteAPI.js';
|
|
2
|
-
import {
|
|
2
|
+
import { getInvalidLocalSyncConfigStatus } from '@/api/diagnostics.js';
|
|
3
|
+
import { BasicRouterRequest, Context, storage } from '@/index.js';
|
|
3
4
|
import { AbstractReplicationJob } from '@/replication/AbstractReplicationJob.js';
|
|
4
5
|
import { AbstractReplicator, AbstractReplicatorOptions, CreateJobOptions } from '@/replication/AbstractReplicator.js';
|
|
6
|
+
import { reprocess } from '@/routes/endpoints/admin.js';
|
|
7
|
+
import { reprocessSyncRules } from '@/routes/endpoints/sync-rules.js';
|
|
5
8
|
import { PersistedReplicationStream } from '@/storage/PersistedReplicationStream.js';
|
|
6
9
|
import type { ReplicationLock } from '@/storage/ReplicationLock.js';
|
|
7
10
|
import { SyncRulesBucketStorage } from '@/storage/SyncRulesBucketStorage.js';
|
|
11
|
+
import { SyncConfigEngine, SyncConfigResultStatus } from '@/system/SyncConfigEngine.js';
|
|
12
|
+
import type { SyncRulesConfig } from '@/util/config/types.js';
|
|
13
|
+
import * as syncConfigParser from '@powersync/service-sync-rules';
|
|
8
14
|
import { describe, expect, it, vi } from 'vitest';
|
|
15
|
+
import { mockServiceContext } from './routes/mocks.js';
|
|
9
16
|
|
|
17
|
+
type TestReplicatorOptions = Omit<AbstractReplicatorOptions, 'syncConfigEngine'> & {
|
|
18
|
+
syncConfigEngine?: SyncConfigEngine;
|
|
19
|
+
};
|
|
10
20
|
class TestReplicator extends AbstractReplicator {
|
|
21
|
+
readonly source: RouteAPI;
|
|
22
|
+
readonly syncConfigEngineForTest: SyncConfigEngine;
|
|
11
23
|
constructor(
|
|
12
24
|
private readonly cleanup: (storage: SyncRulesBucketStorage) => Promise<void>,
|
|
13
|
-
options:
|
|
25
|
+
options: TestReplicatorOptions = {
|
|
14
26
|
id: 'test',
|
|
15
27
|
storageEngine: {} as AbstractReplicatorOptions['storageEngine'],
|
|
16
28
|
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
17
|
-
|
|
29
|
+
configuration: { connections: [], sync_rules: { present: false, exit_on_error: true } },
|
|
18
30
|
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
19
31
|
}
|
|
20
32
|
) {
|
|
21
|
-
|
|
33
|
+
const source = {
|
|
34
|
+
getParseSyncRulesOptions: () => ({
|
|
35
|
+
defaultSchema: 'public',
|
|
36
|
+
connections: Object.fromEntries(
|
|
37
|
+
(options.configuration.connections ?? []).map((c) => [c.tag ?? 'default', { type: c.type }])
|
|
38
|
+
)
|
|
39
|
+
})
|
|
40
|
+
} as RouteAPI;
|
|
41
|
+
const syncConfigEngine =
|
|
42
|
+
options.syncConfigEngine ?? new SyncConfigEngine(options.configuration.sync_rules, () => source);
|
|
43
|
+
super({ ...options, syncConfigEngine });
|
|
44
|
+
this.source = source;
|
|
45
|
+
this.syncConfigEngineForTest = syncConfigEngine;
|
|
22
46
|
}
|
|
23
47
|
|
|
24
48
|
createJob(_options: CreateJobOptions): AbstractReplicationJob {
|
|
@@ -50,25 +74,37 @@ class TestReplicator extends AbstractReplicator {
|
|
|
50
74
|
|
|
51
75
|
async runStartupForTest(): Promise<void> {
|
|
52
76
|
const controller = new AbortController();
|
|
77
|
+
// Keep startup admission real, but skip the continuous replication loop.
|
|
53
78
|
controller.abort();
|
|
54
79
|
(this as any).abortController = controller;
|
|
55
80
|
await (this as any).runLoop();
|
|
56
81
|
}
|
|
57
82
|
|
|
58
|
-
async refreshForTest(
|
|
83
|
+
async refreshForTest(
|
|
84
|
+
configuredLock?: ReplicationLock,
|
|
85
|
+
loadedSyncConfig?: string,
|
|
86
|
+
invalid = false
|
|
87
|
+
): Promise<{ replicationJobStarted: boolean }> {
|
|
59
88
|
(this as any).abortController = new AbortController();
|
|
60
89
|
return (this as any).refresh({
|
|
61
90
|
configuredLock,
|
|
62
|
-
|
|
91
|
+
loadedSyncConfig,
|
|
92
|
+
loadedSyncConfigInvalid: invalid,
|
|
93
|
+
loadedVersionLabel: undefined
|
|
63
94
|
});
|
|
64
95
|
}
|
|
65
96
|
|
|
66
97
|
shouldHandleStreamForTest(
|
|
67
98
|
replicationStream: PersistedReplicationStream,
|
|
68
99
|
loadedSyncRules: string | undefined,
|
|
69
|
-
loadedVersionLabel: string | undefined
|
|
70
|
-
|
|
71
|
-
|
|
100
|
+
loadedVersionLabel: string | undefined,
|
|
101
|
+
invalid = false
|
|
102
|
+
): Promise<boolean> {
|
|
103
|
+
return this.shouldHandleReplicationStream(replicationStream, loadedSyncRules, loadedVersionLabel, invalid);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
get loggerForTest() {
|
|
107
|
+
return this.logger;
|
|
72
108
|
}
|
|
73
109
|
}
|
|
74
110
|
|
|
@@ -91,31 +127,44 @@ describe('AbstractReplicator startup sync config', () => {
|
|
|
91
127
|
const replicator = new TestReplicator(async () => {}, {
|
|
92
128
|
id: 'test',
|
|
93
129
|
storageEngine: {
|
|
94
|
-
activeBucketStorage: {
|
|
130
|
+
activeBucketStorage: {
|
|
131
|
+
configureSyncRules,
|
|
132
|
+
stopProcessingSyncConfigs: vi.fn(async () => {}),
|
|
133
|
+
getActiveSyncConfig: async () => null
|
|
134
|
+
}
|
|
95
135
|
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
96
|
-
configuration: {
|
|
97
|
-
|
|
136
|
+
configuration: {
|
|
137
|
+
connections: [{ type: 'postgresql' }],
|
|
138
|
+
sync_rules: {
|
|
139
|
+
present: true,
|
|
140
|
+
exit_on_error: exitOnError,
|
|
141
|
+
content: yaml
|
|
142
|
+
}
|
|
143
|
+
},
|
|
98
144
|
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
99
145
|
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
100
146
|
});
|
|
147
|
+
const loaded = await replicator.syncConfigEngineForTest.getConfigured();
|
|
148
|
+
expect(getInvalidLocalSyncConfigStatus(loaded, true)?.errors).toEqual(
|
|
149
|
+
expect.arrayContaining([
|
|
150
|
+
expect.objectContaining({
|
|
151
|
+
level: 'fatal',
|
|
152
|
+
message: expect.stringContaining('only supported by MongoDB sources')
|
|
153
|
+
})
|
|
154
|
+
])
|
|
155
|
+
);
|
|
101
156
|
if (exitOnError) {
|
|
102
157
|
await expect(replicator.runStartupForTest()).rejects.toThrow('only supported by MongoDB sources');
|
|
103
158
|
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
104
159
|
} else {
|
|
105
160
|
await expect(replicator.runStartupForTest()).resolves.toBeUndefined();
|
|
106
|
-
expect(configureSyncRules).
|
|
107
|
-
expect(configureSyncRules.mock.calls[0][0].config.plan?.errors).toEqual([
|
|
108
|
-
expect.objectContaining({
|
|
109
|
-
level: 'fatal',
|
|
110
|
-
message: expect.stringContaining('only supported by MongoDB sources')
|
|
111
|
-
})
|
|
112
|
-
]);
|
|
161
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
113
162
|
}
|
|
114
163
|
});
|
|
115
164
|
|
|
116
165
|
it.each(
|
|
117
166
|
[true, false].flatMap((exitOnError) =>
|
|
118
|
-
(['warning', 'valid', 'unexpected error'] as const).map((result) => ({ exitOnError, result }))
|
|
167
|
+
(['warning', 'valid', 'fatal', 'unexpected error'] as const).map((result) => ({ exitOnError, result }))
|
|
119
168
|
)
|
|
120
169
|
)(
|
|
121
170
|
'checks source capabilities before persistence: $result, exit_on_error=$exitOnError',
|
|
@@ -136,32 +185,59 @@ describe('AbstractReplicator startup sync config', () => {
|
|
|
136
185
|
const replicator = new TestReplicator(async () => {}, {
|
|
137
186
|
id: 'test',
|
|
138
187
|
storageEngine: {
|
|
139
|
-
activeBucketStorage: {
|
|
188
|
+
activeBucketStorage: {
|
|
189
|
+
configureSyncRules,
|
|
190
|
+
stopProcessingSyncConfigs: vi.fn(async () => {}),
|
|
191
|
+
getActiveSyncConfig: async () => null
|
|
192
|
+
}
|
|
140
193
|
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
141
|
-
|
|
194
|
+
configuration: {
|
|
195
|
+
connections: [{ type: 'postgresql' }, { type: 'mongodb', tag: 'archive' }],
|
|
196
|
+
sync_rules: {
|
|
197
|
+
present: true,
|
|
198
|
+
exit_on_error: exitOnError,
|
|
199
|
+
content: yaml
|
|
200
|
+
}
|
|
201
|
+
},
|
|
142
202
|
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
143
203
|
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
144
204
|
});
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
205
|
+
using parse = vi.spyOn(syncConfigParser, 'parseSyncConfig');
|
|
206
|
+
using loggedError = vi.spyOn(replicator.loggerForTest, 'error');
|
|
207
|
+
using loggedWarning = vi.spyOn(replicator.loggerForTest, 'warn');
|
|
208
|
+
replicator.source.validateSourceCapabilities = async () => {
|
|
209
|
+
calls.push('validate');
|
|
210
|
+
if (result === 'unexpected error') {
|
|
211
|
+
throw new Error('Source validation failed');
|
|
212
|
+
}
|
|
213
|
+
return result === 'fatal'
|
|
214
|
+
? [{ level: 'fatal', message: 'Source validation failed' }]
|
|
215
|
+
: result === 'warning'
|
|
216
|
+
? [{ level: 'warning', message: 'Missing index' }]
|
|
217
|
+
: [];
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
const rejected = result === 'fatal' || result === 'unexpected error';
|
|
221
|
+
if (rejected && exitOnError) {
|
|
159
222
|
await expect(replicator.runStartupForTest()).rejects.toThrow('Source validation failed');
|
|
160
223
|
expect(calls).toEqual(['validate']);
|
|
161
224
|
} else {
|
|
162
225
|
await replicator.runStartupForTest();
|
|
163
|
-
expect(calls).toEqual(['validate', 'persist']);
|
|
226
|
+
expect(calls).toEqual(rejected ? ['validate'] : ['validate', 'persist']);
|
|
227
|
+
}
|
|
228
|
+
// Source diagnostics must use the same logging levels and prefixes as parser diagnostics.
|
|
229
|
+
if (rejected) {
|
|
230
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
231
|
+
expect(loggedError).toHaveBeenCalledWith('Sync config error: Source validation failed');
|
|
232
|
+
} else if (result === 'warning') {
|
|
233
|
+
expect(loggedWarning).toHaveBeenCalledWith('Sync config warning: Missing index');
|
|
164
234
|
}
|
|
235
|
+
expect(parse).toHaveBeenCalledWith(
|
|
236
|
+
yaml,
|
|
237
|
+
expect.objectContaining({
|
|
238
|
+
connections: { default: { type: 'postgresql' }, archive: { type: 'mongodb' } }
|
|
239
|
+
})
|
|
240
|
+
);
|
|
165
241
|
}
|
|
166
242
|
);
|
|
167
243
|
|
|
@@ -176,9 +252,21 @@ describe('AbstractReplicator startup sync config', () => {
|
|
|
176
252
|
const replicator = new TestReplicator(async () => {}, {
|
|
177
253
|
id: 'test',
|
|
178
254
|
storageEngine: {
|
|
179
|
-
activeBucketStorage: {
|
|
255
|
+
activeBucketStorage: {
|
|
256
|
+
configureSyncRules,
|
|
257
|
+
stopProcessingSyncConfigs: vi.fn(async () => {}),
|
|
258
|
+
getActiveSyncConfig: async () => null
|
|
259
|
+
}
|
|
180
260
|
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
181
|
-
|
|
261
|
+
configuration: {
|
|
262
|
+
connections: [],
|
|
263
|
+
sync_rules: {
|
|
264
|
+
present: true,
|
|
265
|
+
exit_on_error: exitOnError,
|
|
266
|
+
version_label: 'v2',
|
|
267
|
+
content: yaml
|
|
268
|
+
}
|
|
269
|
+
},
|
|
182
270
|
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
183
271
|
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
184
272
|
});
|
|
@@ -211,11 +299,11 @@ describe('AbstractReplicator startup sync config', () => {
|
|
|
211
299
|
});
|
|
212
300
|
|
|
213
301
|
describe('AbstractReplicator heartbeat interval', () => {
|
|
214
|
-
const options:
|
|
302
|
+
const options: TestReplicatorOptions = {
|
|
215
303
|
id: 'test',
|
|
216
304
|
storageEngine: {} as AbstractReplicatorOptions['storageEngine'],
|
|
217
305
|
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
218
|
-
|
|
306
|
+
configuration: { connections: [], sync_rules: { present: false, exit_on_error: true } },
|
|
219
307
|
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
220
308
|
};
|
|
221
309
|
|
|
@@ -322,12 +410,254 @@ describe('AbstractReplicator rolling deployment config matching', () => {
|
|
|
322
410
|
]
|
|
323
411
|
}) as unknown as PersistedReplicationStream;
|
|
324
412
|
|
|
325
|
-
it('requires both YAML and version label to match a processing config', () => {
|
|
413
|
+
it('requires both YAML and version label to match a processing config', async () => {
|
|
326
414
|
const replicator = new TestReplicator(async () => {});
|
|
327
415
|
const stream = processingStream('v2');
|
|
328
416
|
|
|
329
|
-
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v2')).toBe(true);
|
|
330
|
-
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v1')).toBe(false);
|
|
331
|
-
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', undefined)).toBe(false);
|
|
417
|
+
expect(await replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v2')).toBe(true);
|
|
418
|
+
expect(await replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v1')).toBe(false);
|
|
419
|
+
expect(await replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', undefined)).toBe(false);
|
|
420
|
+
});
|
|
421
|
+
});
|
|
422
|
+
|
|
423
|
+
const VALID_CONFIG = /* yaml */ `
|
|
424
|
+
# Active configuration.
|
|
425
|
+
config:
|
|
426
|
+
edition: 3
|
|
427
|
+
streams:
|
|
428
|
+
todos:
|
|
429
|
+
query: SELECT id FROM todos
|
|
430
|
+
`;
|
|
431
|
+
const INVALID_CONFIG = /* yaml */ `
|
|
432
|
+
# Rejected configured sync config.
|
|
433
|
+
config:
|
|
434
|
+
edition: 3
|
|
435
|
+
streams:
|
|
436
|
+
broken:
|
|
437
|
+
query: SELECT FROM
|
|
438
|
+
`;
|
|
439
|
+
|
|
440
|
+
async function makeStartupReplicator(content: string, exitOnError = false, settings?: SyncRulesConfig) {
|
|
441
|
+
const configureSyncRules = vi.fn(async (_options: storage.UpdateSyncRulesOptions) => ({ updated: false }));
|
|
442
|
+
const getActiveSyncConfig = vi.fn();
|
|
443
|
+
const stopProcessingSyncConfigs = vi.fn(async () => {});
|
|
444
|
+
const restartReplication = vi.fn(async (_streamId: number) => {});
|
|
445
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
446
|
+
id: 'test',
|
|
447
|
+
storageEngine: {
|
|
448
|
+
activeBucketStorage: {
|
|
449
|
+
configureSyncRules,
|
|
450
|
+
getActiveSyncConfig,
|
|
451
|
+
stopProcessingSyncConfigs,
|
|
452
|
+
restartReplication,
|
|
453
|
+
getReplicatingReplicationStreams: async () => [],
|
|
454
|
+
getStoppedReplicationStreams: async () => []
|
|
455
|
+
}
|
|
456
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
457
|
+
configuration: {
|
|
458
|
+
connections: [],
|
|
459
|
+
sync_rules: settings ?? { present: true, content, exit_on_error: exitOnError, version_label: 'local' }
|
|
460
|
+
},
|
|
461
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
462
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
463
|
+
});
|
|
464
|
+
return { replicator, configureSyncRules, getActiveSyncConfig, stopProcessingSyncConfigs, restartReplication };
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
describe('invalid configured sync config', () => {
|
|
468
|
+
it('cancels pending work once at startup without cancelling again on refresh', async () => {
|
|
469
|
+
const { replicator, stopProcessingSyncConfigs } = await makeStartupReplicator(INVALID_CONFIG);
|
|
470
|
+
await replicator.runStartupForTest();
|
|
471
|
+
// Later refreshes must leave deployments submitted by a newer runner alone.
|
|
472
|
+
await replicator.refreshForTest(undefined, INVALID_CONFIG, true);
|
|
473
|
+
await replicator.refreshForTest(undefined, INVALID_CONFIG, true);
|
|
474
|
+
expect(stopProcessingSyncConfigs).toHaveBeenCalledOnce();
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
it('does not cancel pending work when exit_on_error is enabled', async () => {
|
|
478
|
+
const { replicator, stopProcessingSyncConfigs } = await makeStartupReplicator(INVALID_CONFIG, true);
|
|
479
|
+
await expect(replicator.runStartupForTest()).rejects.toThrow();
|
|
480
|
+
expect(stopProcessingSyncConfigs).not.toHaveBeenCalled();
|
|
481
|
+
});
|
|
482
|
+
it.each([
|
|
483
|
+
{ inputKind: 'invalid query', yaml: INVALID_CONFIG },
|
|
484
|
+
{ inputKind: 'malformed YAML', yaml: 'streams: [' },
|
|
485
|
+
{ inputKind: 'empty config', yaml: '# empty sync config\n' }
|
|
486
|
+
])('logs and skips rejected YAML without storing it: $inputKind', async ({ yaml }) => {
|
|
487
|
+
const { replicator, configureSyncRules } = await makeStartupReplicator(yaml);
|
|
488
|
+
const errors = vi.spyOn(replicator.loggerForTest, 'error');
|
|
489
|
+
await replicator.runStartupForTest();
|
|
490
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
491
|
+
expect(errors).toHaveBeenCalledWith(expect.stringContaining('Sync config error:'));
|
|
492
|
+
errors.mockRestore();
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
it('uses the same compiled snapshot for deployment and diagnostics', async () => {
|
|
496
|
+
const { replicator, configureSyncRules } = await makeStartupReplicator(VALID_CONFIG);
|
|
497
|
+
await replicator.runStartupForTest();
|
|
498
|
+
const result = await replicator.syncConfigEngineForTest.getConfigured();
|
|
499
|
+
expect(result.status).toBe(SyncConfigResultStatus.ACCEPTED);
|
|
500
|
+
if (result.status === SyncConfigResultStatus.ACCEPTED) {
|
|
501
|
+
expect(configureSyncRules.mock.calls[0][0].config).toBe(result.compiled);
|
|
502
|
+
}
|
|
503
|
+
expect(getInvalidLocalSyncConfigStatus(result, true)).toBeUndefined();
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
it('rejects a file read failure rather than treating it as API mode', async () => {
|
|
507
|
+
const { replicator, configureSyncRules } = await makeStartupReplicator('', false, {
|
|
508
|
+
present: true,
|
|
509
|
+
path: '/nonexistent-powersync-engine-test/sync-config.yaml',
|
|
510
|
+
exit_on_error: false
|
|
511
|
+
});
|
|
512
|
+
const result = await replicator.syncConfigEngineForTest.getConfigured();
|
|
513
|
+
expect(result.status).toBe(SyncConfigResultStatus.UNAVAILABLE);
|
|
514
|
+
// Missing configured input is a startup failure, not permission to use API-managed deployments.
|
|
515
|
+
if (result.status !== SyncConfigResultStatus.UNAVAILABLE) {
|
|
516
|
+
throw new Error('Expected input load failure');
|
|
517
|
+
}
|
|
518
|
+
await expect(replicator.runStartupForTest()).rejects.toBe(result.error);
|
|
519
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
520
|
+
expect(
|
|
521
|
+
getInvalidLocalSyncConfigStatus(await replicator.syncConfigEngineForTest.getConfigured(), true)
|
|
522
|
+
).toBeUndefined();
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it('accepts stored processing before configured content is supplied', async () => {
|
|
526
|
+
const { replicator, configureSyncRules, getActiveSyncConfig } = await makeStartupReplicator('', true, {
|
|
527
|
+
present: false,
|
|
528
|
+
exit_on_error: true
|
|
529
|
+
});
|
|
530
|
+
await replicator.runStartupForTest();
|
|
531
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
532
|
+
const content = {
|
|
533
|
+
sync_rules_content: VALID_CONFIG,
|
|
534
|
+
version_label: 'active',
|
|
535
|
+
syncConfigState: storage.SyncRuleState.PROCESSING
|
|
536
|
+
};
|
|
537
|
+
const stream = { syncConfigContent: [content] } as unknown as PersistedReplicationStream;
|
|
538
|
+
getActiveSyncConfig.mockResolvedValue(null);
|
|
539
|
+
expect(await replicator.shouldHandleStreamForTest(stream, undefined, undefined)).toBe(true);
|
|
540
|
+
expect(getActiveSyncConfig).not.toHaveBeenCalled();
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
it.each([
|
|
544
|
+
{ inputKind: 'invalid query', content: INVALID_CONFIG },
|
|
545
|
+
{ inputKind: 'empty config', content: '# empty sync config\n' }
|
|
546
|
+
])('still fails startup with exit_on_error enabled: $inputKind', async ({ content }) => {
|
|
547
|
+
const { replicator, configureSyncRules } = await makeStartupReplicator(content, true);
|
|
548
|
+
await expect(replicator.runStartupForTest()).rejects.toThrow();
|
|
549
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
it('persists valid config, including warnings', async () => {
|
|
553
|
+
const { replicator, configureSyncRules } = await makeStartupReplicator(VALID_CONFIG);
|
|
554
|
+
await replicator.runStartupForTest();
|
|
555
|
+
expect(configureSyncRules).toHaveBeenCalledExactlyOnceWith(
|
|
556
|
+
expect.objectContaining({
|
|
557
|
+
version_label: 'local',
|
|
558
|
+
config: expect.objectContaining({ yaml: VALID_CONFIG })
|
|
559
|
+
})
|
|
560
|
+
);
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Previously persisted invalid work remains owned only when both local YAML and label match.
|
|
565
|
+
* This compatibility path also works without an active config (a rejected first deployment).
|
|
566
|
+
*/
|
|
567
|
+
it.each([{ deploymentMatch: 'matching' }, { deploymentMatch: 'other-yaml' }, { deploymentMatch: 'other-label' }])(
|
|
568
|
+
'checks ownership of previously persisted invalid config: $deploymentMatch',
|
|
569
|
+
async ({ deploymentMatch }) => {
|
|
570
|
+
const { replicator, getActiveSyncConfig } = await makeStartupReplicator(INVALID_CONFIG);
|
|
571
|
+
getActiveSyncConfig.mockResolvedValue(null);
|
|
572
|
+
const stream = {
|
|
573
|
+
syncConfigContent: [
|
|
574
|
+
{
|
|
575
|
+
syncConfigState: storage.SyncRuleState.PROCESSING,
|
|
576
|
+
sync_rules_content: deploymentMatch === 'other-yaml' ? VALID_CONFIG : INVALID_CONFIG,
|
|
577
|
+
version_label: deploymentMatch === 'other-label' ? 'another-worker' : 'local'
|
|
578
|
+
}
|
|
579
|
+
]
|
|
580
|
+
} as unknown as PersistedReplicationStream;
|
|
581
|
+
expect(await replicator.shouldHandleStreamForTest(stream, INVALID_CONFIG, 'local', true)).toBe(
|
|
582
|
+
deploymentMatch === 'matching'
|
|
583
|
+
);
|
|
584
|
+
if (deploymentMatch === 'matching') {
|
|
585
|
+
expect(getActiveSyncConfig).not.toHaveBeenCalled();
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
);
|
|
589
|
+
|
|
590
|
+
it.each([
|
|
591
|
+
{ recoveryMatch: 'matching' },
|
|
592
|
+
{ recoveryMatch: 'other-yaml' },
|
|
593
|
+
{ recoveryMatch: 'other-label' },
|
|
594
|
+
{ recoveryMatch: 'other-storage' },
|
|
595
|
+
{ recoveryMatch: 'no-active' }
|
|
596
|
+
])('only accepts recovery matching the active config: $recoveryMatch', async ({ recoveryMatch }) => {
|
|
597
|
+
const { replicator, getActiveSyncConfig } = await makeStartupReplicator(INVALID_CONFIG);
|
|
598
|
+
const active = { sync_rules_content: VALID_CONFIG, version_label: 'active', storageVersion: 2 };
|
|
599
|
+
getActiveSyncConfig.mockResolvedValue(recoveryMatch == 'no-active' ? null : { content: active });
|
|
600
|
+
const processing = {
|
|
601
|
+
...active,
|
|
602
|
+
syncConfigState: storage.SyncRuleState.PROCESSING,
|
|
603
|
+
...(recoveryMatch == 'other-yaml' ? { sync_rules_content: INVALID_CONFIG } : {}),
|
|
604
|
+
...(recoveryMatch == 'other-label' ? { version_label: 'different' } : {}),
|
|
605
|
+
// A storage upgrade does not change which YAML deployment the replacement belongs to.
|
|
606
|
+
...(recoveryMatch == 'other-storage' ? { storageVersion: 4 } : {})
|
|
607
|
+
};
|
|
608
|
+
const stream = { syncConfigContent: [processing] } as unknown as PersistedReplicationStream;
|
|
609
|
+
expect(await replicator.shouldHandleStreamForTest(stream, INVALID_CONFIG, 'local', true)).toBe(
|
|
610
|
+
recoveryMatch == 'matching' || recoveryMatch == 'other-storage'
|
|
611
|
+
);
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
it.each([{ route: reprocess }, { route: reprocessSyncRules }])(
|
|
615
|
+
'accepts a storage upgrade requested by $route.path',
|
|
616
|
+
async ({ route }) => {
|
|
617
|
+
const { replicator, getActiveSyncConfig } = await makeStartupReplicator(INVALID_CONFIG);
|
|
618
|
+
const active = { sync_rules_content: VALID_CONFIG, version_label: 'active', storageVersion: 2 };
|
|
619
|
+
getActiveSyncConfig.mockResolvedValue({ content: active });
|
|
620
|
+
let replacement: PersistedReplicationStream | undefined;
|
|
621
|
+
const updateSyncRules = vi.fn(async (options: storage.UpdateSyncRulesOptions) => {
|
|
622
|
+
// Model the backend selecting its v4 default for YAML without a storage_version.
|
|
623
|
+
expect(options.storageVersion).toBeUndefined();
|
|
624
|
+
expect(options.config.parsed.config.storageVersion).toBeUndefined();
|
|
625
|
+
replacement = {
|
|
626
|
+
replicationStreamName: 'replacement',
|
|
627
|
+
syncConfigContent: [
|
|
628
|
+
{
|
|
629
|
+
syncConfigState: storage.SyncRuleState.PROCESSING,
|
|
630
|
+
sync_rules_content: options.config.yaml,
|
|
631
|
+
version_label: options.version_label,
|
|
632
|
+
storageVersion: 4
|
|
633
|
+
}
|
|
634
|
+
]
|
|
635
|
+
} as unknown as PersistedReplicationStream;
|
|
636
|
+
return replacement;
|
|
637
|
+
});
|
|
638
|
+
// A separate API runner submits the rebuild; the replicator still holds rejected configured YAML.
|
|
639
|
+
const service = mockServiceContext(null);
|
|
640
|
+
Object.assign(service.storageEngine.activeBucketStorage, {
|
|
641
|
+
getActiveSyncConfig,
|
|
642
|
+
getDeployingSyncConfig: async () => null,
|
|
643
|
+
updateSyncRules
|
|
644
|
+
});
|
|
645
|
+
await route.handler({
|
|
646
|
+
context: { service_context: service } as Context,
|
|
647
|
+
request: { headers: {}, hostname: '', protocol: 'http' } as BasicRouterRequest,
|
|
648
|
+
params: {}
|
|
649
|
+
});
|
|
650
|
+
expect(updateSyncRules).toHaveBeenCalledOnce();
|
|
651
|
+
expect(await replicator.shouldHandleStreamForTest(replacement!, INVALID_CONFIG, 'local', true)).toBe(true);
|
|
652
|
+
}
|
|
653
|
+
);
|
|
654
|
+
|
|
655
|
+
it('keeps active-only streams running without consulting recovery state', async () => {
|
|
656
|
+
const { replicator, getActiveSyncConfig } = await makeStartupReplicator(INVALID_CONFIG);
|
|
657
|
+
const stream = {
|
|
658
|
+
syncConfigContent: [{ syncConfigState: storage.SyncRuleState.ACTIVE }]
|
|
659
|
+
} as unknown as PersistedReplicationStream;
|
|
660
|
+
expect(await replicator.shouldHandleStreamForTest(stream, INVALID_CONFIG, 'local', true)).toBe(true);
|
|
661
|
+
expect(getActiveSyncConfig).not.toHaveBeenCalled();
|
|
332
662
|
});
|
|
333
663
|
});
|
|
@@ -2,9 +2,9 @@ import {
|
|
|
2
2
|
DEFAULT_HYDRATION_STATE,
|
|
3
3
|
DEFAULT_TAG,
|
|
4
4
|
nodeSqlite,
|
|
5
|
+
parseSyncConfig,
|
|
5
6
|
PrecompiledSyncConfig,
|
|
6
|
-
serializeSyncPlan
|
|
7
|
-
SqlSyncRules
|
|
7
|
+
serializeSyncPlan
|
|
8
8
|
} from '@powersync/service-sync-rules';
|
|
9
9
|
import * as sqlite from 'node:sqlite';
|
|
10
10
|
import { describe, expect, test } from 'vitest';
|
|
@@ -32,7 +32,7 @@ event_definitions:
|
|
|
32
32
|
|
|
33
33
|
describe('persisted compiled replication events', () => {
|
|
34
34
|
test('dual-writes additive compiled events and restores the compiled evaluator', () => {
|
|
35
|
-
const parsed =
|
|
35
|
+
const parsed = parseSyncConfig(yamlWithEvents, { defaultSchema: 'test_schema' });
|
|
36
36
|
const update = updateSyncRulesFromConfig(parsed);
|
|
37
37
|
const compiled = update.config.plan!;
|
|
38
38
|
const originalEvent = compiled.plan.events![0];
|
|
@@ -87,7 +87,7 @@ describe('persisted compiled replication events', () => {
|
|
|
87
87
|
});
|
|
88
88
|
|
|
89
89
|
test('restores raw event descriptors attached to version 1 and 2 plans', () => {
|
|
90
|
-
const parsed =
|
|
90
|
+
const parsed = parseSyncConfig(yamlWithoutEvents, { defaultSchema: 'test_schema' });
|
|
91
91
|
const update = updateSyncRulesFromConfig(parsed);
|
|
92
92
|
const legacy: SerializedSyncPlan = {
|
|
93
93
|
...update.config.plan!,
|
|
@@ -116,7 +116,7 @@ describe('persisted compiled replication events', () => {
|
|
|
116
116
|
['request parameters', 'SELECT user_id FROM checkpoints WHERE user_id = request.user_id()'],
|
|
117
117
|
['unsupported expressions', 'SELECT user_id FROM checkpoints WHERE active IN (1, 2)']
|
|
118
118
|
])('loads legacy event descriptors without validating ignored %s', (_description, eventQuery) => {
|
|
119
|
-
const parsed =
|
|
119
|
+
const parsed = parseSyncConfig(yamlWithoutEvents, { defaultSchema: 'test_schema' });
|
|
120
120
|
const update = updateSyncRulesFromConfig(parsed);
|
|
121
121
|
const legacy: SerializedSyncPlan = {
|
|
122
122
|
...update.config.plan!,
|