@powersync/service-core 1.27.0 → 1.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/dist/api/RouteAPI.d.ts +8 -1
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.js +33 -16
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/source-capabilities.d.ts +7 -0
- package/dist/api/source-capabilities.js +23 -0
- package/dist/api/source-capabilities.js.map +1 -0
- package/dist/auth/JwtPayload.d.ts +2 -2
- package/dist/auth/JwtPayload.js +2 -2
- package/dist/auth/JwtPayload.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +4 -6
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +15 -0
- package/dist/replication/AbstractReplicator.js +35 -3
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.js +13 -2
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +39 -27
- package/dist/routes/endpoints/admin.d.ts +78 -54
- package/dist/routes/endpoints/admin.js +23 -9
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +39 -6
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +11 -1
- package/dist/storage/BucketStorageFactory.js +22 -2
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.d.ts +2 -1
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +3 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +1 -1
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +5 -1
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +9 -1
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +30 -14
- package/src/api/source-capabilities.ts +24 -0
- package/src/auth/JwtPayload.ts +2 -2
- package/src/replication/AbstractReplicationJob.ts +2 -4
- package/src/replication/AbstractReplicator.ts +44 -3
- package/src/replication/ReplicationModule.ts +12 -2
- package/src/routes/endpoints/admin.ts +28 -10
- package/src/routes/endpoints/sync-rules.ts +42 -9
- package/src/storage/BucketStorageFactory.ts +22 -2
- package/src/storage/PersistedSyncConfigContent.ts +3 -0
- package/src/storage/StorageProvider.ts +3 -0
- package/src/storage/SyncRulesBucketStorage.ts +1 -1
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +13 -1
- package/test/src/AbstractReplicator.test.ts +156 -0
- package/test/src/diagnostics.test.ts +128 -0
- package/test/src/routes/admin.test.ts +235 -4
- package/test/src/routes/sync-rules.test.ts +124 -0
- package/test/src/source_table_config.test.ts +104 -0
- package/test/src/storage/BucketStorageFactory.test.ts +46 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -8,11 +8,12 @@ import {
|
|
|
8
8
|
SqlSyncRules,
|
|
9
9
|
StaticSchema
|
|
10
10
|
} from '@powersync/service-sync-rules';
|
|
11
|
-
import { internal_routes } from '@powersync/service-types';
|
|
11
|
+
import { internal_routes, ServiceInfo } from '@powersync/service-types';
|
|
12
12
|
|
|
13
13
|
import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
|
|
14
14
|
import * as api from '../../api/api-index.js';
|
|
15
15
|
import * as storage from '../../storage/storage-index.js';
|
|
16
|
+
import { POWERSYNC_VERSION } from '../../util/version.js';
|
|
16
17
|
import { authApi } from '../auth.js';
|
|
17
18
|
import { routeDefinition } from '../router.js';
|
|
18
19
|
|
|
@@ -58,19 +59,28 @@ export const diagnostics = routeDefinition({
|
|
|
58
59
|
const { context } = payload;
|
|
59
60
|
const { service_context } = context;
|
|
60
61
|
const include_content = payload.params.sync_rules_content ?? false;
|
|
62
|
+
const serviceResponse: { service: ServiceInfo } = {
|
|
63
|
+
service: {
|
|
64
|
+
version: POWERSYNC_VERSION
|
|
65
|
+
}
|
|
66
|
+
};
|
|
61
67
|
|
|
62
68
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
63
69
|
|
|
64
70
|
const status = await apiHandler.getConnectionStatus();
|
|
65
71
|
if (!status) {
|
|
66
72
|
return internal_routes.DiagnosticsResponse.encode({
|
|
73
|
+
...serviceResponse,
|
|
67
74
|
connections: []
|
|
68
75
|
});
|
|
69
76
|
}
|
|
70
77
|
|
|
71
78
|
const {
|
|
72
|
-
storageEngine: { activeBucketStorage }
|
|
79
|
+
storageEngine: { activeBucketStorage },
|
|
80
|
+
configuration
|
|
73
81
|
} = service_context;
|
|
82
|
+
|
|
83
|
+
const sync_config_source = configuration?.sync_rules.present ? 'config' : 'api';
|
|
74
84
|
const active = await activeBucketStorage.getActiveSyncConfig();
|
|
75
85
|
const deploying = await activeBucketStorage.getDeployingSyncConfig();
|
|
76
86
|
|
|
@@ -102,6 +112,7 @@ export const diagnostics = routeDefinition({
|
|
|
102
112
|
);
|
|
103
113
|
|
|
104
114
|
return internal_routes.DiagnosticsResponse.encode({
|
|
115
|
+
...serviceResponse,
|
|
105
116
|
connections: [
|
|
106
117
|
{
|
|
107
118
|
...status,
|
|
@@ -110,7 +121,8 @@ export const diagnostics = routeDefinition({
|
|
|
110
121
|
}
|
|
111
122
|
],
|
|
112
123
|
active_sync_rules: active_status,
|
|
113
|
-
deploying_sync_rules: deploying_status
|
|
124
|
+
deploying_sync_rules: deploying_status,
|
|
125
|
+
sync_config_source
|
|
114
126
|
});
|
|
115
127
|
}
|
|
116
128
|
});
|
|
@@ -161,11 +173,18 @@ export const reprocess = routeDefinition({
|
|
|
161
173
|
// 1. This always re-parses the source YAML. If there are changes to the Sync Streams compiler, that can affect the sync plan.
|
|
162
174
|
// 2. If the source does not set the storage version, this will update it do the current version.
|
|
163
175
|
// We can consider tweaking this behavior in the future.
|
|
176
|
+
const parsed = SqlSyncRules.fromYaml(active.content.sync_rules_content, {
|
|
177
|
+
...apiHandler.getParseSyncRulesOptions(),
|
|
178
|
+
schema: undefined,
|
|
179
|
+
// This sync config already passed validation. But if the config is not valid anymore due
|
|
180
|
+
// to a service change, we do want to report the error here.
|
|
181
|
+
throwOnError: true
|
|
182
|
+
});
|
|
183
|
+
storage.logSyncConfigErrors(parsed);
|
|
184
|
+
await api.assertSourceCapabilities(apiHandler, parsed.config);
|
|
185
|
+
|
|
164
186
|
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
165
|
-
storage.
|
|
166
|
-
// This sync config already passed validation. But if the config is not valid anymore due
|
|
167
|
-
// to a service change, we do want to report the error here.
|
|
168
|
-
validate: true,
|
|
187
|
+
storage.updateSyncRulesFromConfig(parsed, {
|
|
169
188
|
version_label: active.content.version_label,
|
|
170
189
|
forceNewReplicationStream: true
|
|
171
190
|
})
|
|
@@ -188,7 +207,6 @@ export const reprocess = routeDefinition({
|
|
|
188
207
|
|
|
189
208
|
class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigContent {
|
|
190
209
|
constructor(
|
|
191
|
-
private readonly apiHandler: api.RouteAPI,
|
|
192
210
|
private readonly schema: SourceSchema,
|
|
193
211
|
data: storage.PersistedSyncConfigContentData
|
|
194
212
|
) {
|
|
@@ -197,7 +215,7 @@ class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigConte
|
|
|
197
215
|
|
|
198
216
|
parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
199
217
|
const syncConfig = SqlSyncRules.fromYaml(this.sync_rules_content, {
|
|
200
|
-
...
|
|
218
|
+
...options,
|
|
201
219
|
schema: this.schema
|
|
202
220
|
});
|
|
203
221
|
|
|
@@ -237,7 +255,7 @@ export const validate = routeDefinition({
|
|
|
237
255
|
const schemaData = await api.getConnectionsSchema(apiHandler);
|
|
238
256
|
const schema = new StaticSchema(schemaData.connections);
|
|
239
257
|
|
|
240
|
-
const sync_rules = new FakeSyncRulesContentForValidation(
|
|
258
|
+
const sync_rules = new FakeSyncRulesContentForValidation(schema, {
|
|
241
259
|
// Dummy values
|
|
242
260
|
replicationStreamId: 0,
|
|
243
261
|
replicationStreamName: '',
|
|
@@ -2,9 +2,10 @@ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-frame
|
|
|
2
2
|
import { SqlSyncRules, SyncConfigWithErrors, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
3
3
|
import type { FastifyPluginAsync } from 'fastify';
|
|
4
4
|
import * as t from 'ts-codec';
|
|
5
|
+
import { assertSourceCapabilities } from '../../api/source-capabilities.js';
|
|
5
6
|
|
|
6
|
-
import { RouteAPI } from '../../api/RouteAPI.js';
|
|
7
|
-
import {
|
|
7
|
+
import { PatternResult, RouteAPI } from '../../api/RouteAPI.js';
|
|
8
|
+
import { logSyncConfigErrors, updateSyncRulesFromConfig } from '../../storage/BucketStorageFactory.js';
|
|
8
9
|
import { authApi } from '../auth.js';
|
|
9
10
|
import { routeDefinition } from '../router.js';
|
|
10
11
|
|
|
@@ -55,6 +56,7 @@ export const deploySyncRules = routeDefinition({
|
|
|
55
56
|
let syncConfig: SyncConfigWithErrors;
|
|
56
57
|
|
|
57
58
|
try {
|
|
59
|
+
// First ensure the SyncConfig parses correctly
|
|
58
60
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
59
61
|
syncConfig = SqlSyncRules.fromYaml(content, {
|
|
60
62
|
...apiHandler.getParseSyncRulesOptions(),
|
|
@@ -70,6 +72,10 @@ export const deploySyncRules = routeDefinition({
|
|
|
70
72
|
});
|
|
71
73
|
}
|
|
72
74
|
|
|
75
|
+
logSyncConfigErrors(syncConfig);
|
|
76
|
+
// Validate if the current sources support the configuration specified in the sync config
|
|
77
|
+
await assertSourceCapabilities(service_context.routerEngine.getAPI(), syncConfig.config);
|
|
78
|
+
|
|
73
79
|
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
|
|
74
80
|
|
|
75
81
|
return {
|
|
@@ -176,11 +182,18 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
176
182
|
}
|
|
177
183
|
|
|
178
184
|
const sync_rules = active.content;
|
|
185
|
+
const parsed = SqlSyncRules.fromYaml(sync_rules.sync_rules_content, {
|
|
186
|
+
...payload.context.service_context.routerEngine.getAPI().getParseSyncRulesOptions(),
|
|
187
|
+
schema: undefined,
|
|
188
|
+
// This sync config already passed validation. But if the rules are not valid anymore due
|
|
189
|
+
// to a service change, we do want to report the error here.
|
|
190
|
+
throwOnError: true
|
|
191
|
+
});
|
|
192
|
+
logSyncConfigErrors(parsed);
|
|
193
|
+
await assertSourceCapabilities(payload.context.service_context.routerEngine.getAPI(), parsed.config);
|
|
194
|
+
|
|
179
195
|
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
180
|
-
|
|
181
|
-
// This sync config already passed validation. But if the rules are not valid anymore due
|
|
182
|
-
// to a service change, we do want to report the error here.
|
|
183
|
-
validate: true,
|
|
196
|
+
updateSyncRulesFromConfig(parsed, {
|
|
184
197
|
version_label: sync_rules.version_label,
|
|
185
198
|
forceNewReplicationStream: true
|
|
186
199
|
})
|
|
@@ -208,11 +221,31 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
|
|
|
208
221
|
// No schema-based validation at this point
|
|
209
222
|
schema: undefined
|
|
210
223
|
});
|
|
211
|
-
const
|
|
212
|
-
const
|
|
224
|
+
const errors: string[] = [];
|
|
225
|
+
const warnings = rules.errors.filter((error) => error.type === 'warning').map((error) => error.message);
|
|
226
|
+
const recordError = (error: any) => {
|
|
227
|
+
errors.push(...(error instanceof SyncRulesErrors ? error.errors.map((e) => e.message) : [error.message]));
|
|
228
|
+
};
|
|
229
|
+
try {
|
|
230
|
+
const diagnostics = (await apiHandler.validateSourceCapabilities?.(rules.config)) ?? [];
|
|
231
|
+
for (const diagnostic of diagnostics) {
|
|
232
|
+
(diagnostic.level === 'fatal' ? errors : warnings).push(diagnostic.message);
|
|
233
|
+
}
|
|
234
|
+
} catch (e) {
|
|
235
|
+
recordError(e);
|
|
236
|
+
}
|
|
237
|
+
let resolved_tables: PatternResult[] = [];
|
|
238
|
+
try {
|
|
239
|
+
const source_table_patterns = rules.config.getSourceTables();
|
|
240
|
+
resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);
|
|
241
|
+
} catch (e) {
|
|
242
|
+
recordError(e);
|
|
243
|
+
}
|
|
213
244
|
|
|
214
245
|
return {
|
|
215
|
-
valid:
|
|
246
|
+
valid: errors.length == 0,
|
|
247
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
248
|
+
...(errors.length > 0 ? { errors } : {}),
|
|
216
249
|
bucket_definitions: rules.config.debugRepresentation(),
|
|
217
250
|
source_tables: resolved_tables,
|
|
218
251
|
data_tables: rules.config.debugGetOutputTables()
|
|
@@ -227,14 +227,15 @@ export function updateSyncRulesFromYaml(
|
|
|
227
227
|
content: string,
|
|
228
228
|
options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
|
|
229
229
|
): UpdateSyncRulesOptions {
|
|
230
|
+
const { validate, ...updateOptions } = options ?? {};
|
|
230
231
|
const config = SqlSyncRules.fromYaml(content, {
|
|
231
232
|
// No schema-based validation at this point
|
|
232
233
|
schema: undefined,
|
|
233
234
|
defaultSchema: options?.defaultSchema ?? 'not_applicable', // Not needed for validation
|
|
234
|
-
throwOnError:
|
|
235
|
+
throwOnError: validate ?? false
|
|
235
236
|
});
|
|
236
237
|
|
|
237
|
-
return updateSyncRulesFromConfig(config,
|
|
238
|
+
return updateSyncRulesFromConfig(config, updateOptions);
|
|
238
239
|
}
|
|
239
240
|
|
|
240
241
|
export function updateSyncRulesFromConfig(
|
|
@@ -261,6 +262,25 @@ export function updateSyncRulesFromConfig(
|
|
|
261
262
|
return { config: { yaml: config.content, plan, parsed }, ...options };
|
|
262
263
|
}
|
|
263
264
|
|
|
265
|
+
/**
|
|
266
|
+
* Logs the errors and warnings of a parsed sync config
|
|
267
|
+
*
|
|
268
|
+
* Call this where a sync config is loaded or deployed, so that operators see these in the service logs and not only
|
|
269
|
+
* in the diagnostics and validate API responses.
|
|
270
|
+
*
|
|
271
|
+
* Fatal errors usually throw during parsing before this is reached. They can reach this when the sync config is parsed
|
|
272
|
+
* without validation, such as at startup with `exit_on_error: false`.
|
|
273
|
+
*/
|
|
274
|
+
export function logSyncConfigErrors(parsed: SyncConfigWithErrors, log: typeof logger = logger) {
|
|
275
|
+
for (const error of parsed.errors) {
|
|
276
|
+
if (error.type == 'warning') {
|
|
277
|
+
log.warn(`Sync config warning: ${error.message}`);
|
|
278
|
+
} else {
|
|
279
|
+
log.error(`Sync config error: ${error.message}`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
264
284
|
export interface GetIntanceOptions {
|
|
265
285
|
/**
|
|
266
286
|
* The job lease, including a lease acquired during initial configuration.
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
HydrationState,
|
|
11
11
|
nodeSqlite,
|
|
12
12
|
PrecompiledSyncConfig,
|
|
13
|
+
SourceConnectionMap,
|
|
13
14
|
SqlSyncRules,
|
|
14
15
|
SyncConfigWithErrors,
|
|
15
16
|
versionedHydrationState,
|
|
@@ -202,6 +203,8 @@ export interface PersistedSyncConfigContentData {
|
|
|
202
203
|
readonly version_label?: string;
|
|
203
204
|
}
|
|
204
205
|
export type PersistedSyncConfigId = string;
|
|
206
|
+
|
|
205
207
|
export interface ParseSyncConfigOptions {
|
|
206
208
|
defaultSchema: string;
|
|
209
|
+
connections?: SourceConnectionMap;
|
|
207
210
|
}
|
|
@@ -11,7 +11,7 @@ import * as util from '../util/util-index.js';
|
|
|
11
11
|
import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
|
|
12
12
|
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
13
13
|
import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
|
|
14
|
-
import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
|
|
14
|
+
import type { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
|
|
15
15
|
import { SourceEntityDescriptor } from './SourceEntity.js';
|
|
16
16
|
import { SourceTable } from './SourceTable.js';
|
|
17
17
|
import { SourceTableCandidateReconciler } from './SourceTableReconciler.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CompatibilityContext } from '@powersync/service-sync-rules';
|
|
1
|
+
import { CompatibilityContext, sourceTableConfigsEqual } from '@powersync/service-sync-rules';
|
|
2
2
|
import { Logger } from 'winston';
|
|
3
3
|
import { SerializedSyncPlan, UpdateSyncRulesOptions } from '../BucketStorageFactory.js';
|
|
4
4
|
|
|
@@ -37,5 +37,17 @@ export function isCompatible(
|
|
|
37
37
|
return false;
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
if (
|
|
41
|
+
existingPlans.some(
|
|
42
|
+
(existing) =>
|
|
43
|
+
!sourceTableConfigsEqual(
|
|
44
|
+
existing!.plan.sourceTableConfig ?? {},
|
|
45
|
+
updateConfig.plan!.plan.sourceTableConfig ?? {}
|
|
46
|
+
)
|
|
47
|
+
)
|
|
48
|
+
) {
|
|
49
|
+
logger.info('Source table configuration changed - incremental reprocessing not supported');
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
40
52
|
return true;
|
|
41
53
|
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import type { RouteAPI } from '@/api/RouteAPI.js';
|
|
2
|
+
import { assertSourceCapabilities } from '@/api/source-capabilities.js';
|
|
1
3
|
import { AbstractReplicationJob } from '@/replication/AbstractReplicationJob.js';
|
|
2
4
|
import { AbstractReplicator, AbstractReplicatorOptions, CreateJobOptions } from '@/replication/AbstractReplicator.js';
|
|
3
5
|
import { PersistedReplicationStream } from '@/storage/PersistedReplicationStream.js';
|
|
6
|
+
import type { ReplicationLock } from '@/storage/ReplicationLock.js';
|
|
4
7
|
import { SyncRulesBucketStorage } from '@/storage/SyncRulesBucketStorage.js';
|
|
5
8
|
import { describe, expect, it, vi } from 'vitest';
|
|
6
9
|
|
|
@@ -45,6 +48,21 @@ class TestReplicator extends AbstractReplicator {
|
|
|
45
48
|
return (this as any).heartbeatIntervalNanos;
|
|
46
49
|
}
|
|
47
50
|
|
|
51
|
+
async runStartupForTest(): Promise<void> {
|
|
52
|
+
const controller = new AbortController();
|
|
53
|
+
controller.abort();
|
|
54
|
+
(this as any).abortController = controller;
|
|
55
|
+
await (this as any).runLoop();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async refreshForTest(configuredLock?: ReplicationLock): Promise<{ replicationJobStarted: boolean }> {
|
|
59
|
+
(this as any).abortController = new AbortController();
|
|
60
|
+
return (this as any).refresh({
|
|
61
|
+
configuredLock,
|
|
62
|
+
loadedVersionLabel: this.syncRuleProvider.versionLabel
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
|
|
48
66
|
shouldHandleStreamForTest(
|
|
49
67
|
replicationStream: PersistedReplicationStream,
|
|
50
68
|
loadedSyncRules: string | undefined,
|
|
@@ -54,6 +72,144 @@ class TestReplicator extends AbstractReplicator {
|
|
|
54
72
|
}
|
|
55
73
|
}
|
|
56
74
|
|
|
75
|
+
describe('AbstractReplicator startup sync config', () => {
|
|
76
|
+
it.each([true, false])('respects exit_on_error=%s for unsupported file-loaded filters', async (exitOnError) => {
|
|
77
|
+
const yaml = /* yaml */ `
|
|
78
|
+
# sync config fixture.
|
|
79
|
+
config:
|
|
80
|
+
edition: 3
|
|
81
|
+
source_table_options:
|
|
82
|
+
orders:
|
|
83
|
+
mongodb_filter_expression: { $eq: ['$$doc.active', true] }
|
|
84
|
+
streams:
|
|
85
|
+
orders:
|
|
86
|
+
query: SELECT * FROM orders
|
|
87
|
+
`;
|
|
88
|
+
const configureSyncRules = vi.fn(
|
|
89
|
+
async (_options: import('@/storage/BucketStorageFactory.js').UpdateSyncRulesOptions) => ({ updated: false })
|
|
90
|
+
);
|
|
91
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
92
|
+
id: 'test',
|
|
93
|
+
storageEngine: {
|
|
94
|
+
activeBucketStorage: { configureSyncRules }
|
|
95
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
96
|
+
configuration: { connections: [{ type: 'postgresql' }] },
|
|
97
|
+
syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: undefined },
|
|
98
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
99
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
100
|
+
});
|
|
101
|
+
if (exitOnError) {
|
|
102
|
+
await expect(replicator.runStartupForTest()).rejects.toThrow('only supported by MongoDB sources');
|
|
103
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
104
|
+
} else {
|
|
105
|
+
await expect(replicator.runStartupForTest()).resolves.toBeUndefined();
|
|
106
|
+
expect(configureSyncRules).toHaveBeenCalledOnce();
|
|
107
|
+
expect(configureSyncRules.mock.calls[0][0].config.plan?.errors).toEqual([
|
|
108
|
+
expect.objectContaining({
|
|
109
|
+
level: 'fatal',
|
|
110
|
+
message: expect.stringContaining('only supported by MongoDB sources')
|
|
111
|
+
})
|
|
112
|
+
]);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it.each(
|
|
117
|
+
[true, false].flatMap((exitOnError) =>
|
|
118
|
+
(['warning', 'valid', 'unexpected error'] as const).map((result) => ({ exitOnError, result }))
|
|
119
|
+
)
|
|
120
|
+
)(
|
|
121
|
+
'checks source capabilities before persistence: $result, exit_on_error=$exitOnError',
|
|
122
|
+
async ({ result, exitOnError }) => {
|
|
123
|
+
const yaml = /* yaml */ `
|
|
124
|
+
# sync config fixture.
|
|
125
|
+
config:
|
|
126
|
+
edition: 3
|
|
127
|
+
streams:
|
|
128
|
+
orders:
|
|
129
|
+
query: SELECT * FROM orders
|
|
130
|
+
`;
|
|
131
|
+
const calls: string[] = [];
|
|
132
|
+
const configureSyncRules = vi.fn(async () => {
|
|
133
|
+
calls.push('persist');
|
|
134
|
+
return { updated: false };
|
|
135
|
+
});
|
|
136
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
137
|
+
id: 'test',
|
|
138
|
+
storageEngine: {
|
|
139
|
+
activeBucketStorage: { configureSyncRules }
|
|
140
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
141
|
+
syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: undefined },
|
|
142
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
143
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
144
|
+
});
|
|
145
|
+
replicator.registerSourceCapabilitiesAssertion((config) =>
|
|
146
|
+
assertSourceCapabilities(
|
|
147
|
+
{
|
|
148
|
+
async validateSourceCapabilities() {
|
|
149
|
+
calls.push('validate');
|
|
150
|
+
if (result === 'unexpected error') throw new Error('Source validation failed');
|
|
151
|
+
return result === 'warning' ? [{ level: 'warning', message: 'Missing index' }] : [];
|
|
152
|
+
}
|
|
153
|
+
} as unknown as RouteAPI,
|
|
154
|
+
config
|
|
155
|
+
)
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
if (result === 'unexpected error' && exitOnError) {
|
|
159
|
+
await expect(replicator.runStartupForTest()).rejects.toThrow('Source validation failed');
|
|
160
|
+
expect(calls).toEqual(['validate']);
|
|
161
|
+
} else {
|
|
162
|
+
await replicator.runStartupForTest();
|
|
163
|
+
expect(calls).toEqual(['validate', 'persist']);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
it.each([true, false])('retains source-table options with exit_on_error=%s', async (exitOnError) => {
|
|
169
|
+
const yaml = /* yaml */ `
|
|
170
|
+
{
|
|
171
|
+
config: { edition: 3, source_table_options: { orders: { mongodb_filter_expression: 'disabled' } } },
|
|
172
|
+
streams: { orders: { query: SELECT * FROM orders } }
|
|
173
|
+
}
|
|
174
|
+
`;
|
|
175
|
+
const configureSyncRules = vi.fn(async () => ({ updated: false }));
|
|
176
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
177
|
+
id: 'test',
|
|
178
|
+
storageEngine: {
|
|
179
|
+
activeBucketStorage: { configureSyncRules }
|
|
180
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
181
|
+
syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: 'v2' },
|
|
182
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
183
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
await replicator.runStartupForTest();
|
|
187
|
+
|
|
188
|
+
expect(configureSyncRules).toHaveBeenCalledExactlyOnceWith(
|
|
189
|
+
expect.objectContaining({
|
|
190
|
+
lock: true,
|
|
191
|
+
version_label: 'v2',
|
|
192
|
+
config: expect.objectContaining({
|
|
193
|
+
parsed: expect.objectContaining({
|
|
194
|
+
errors: [],
|
|
195
|
+
config: expect.objectContaining({
|
|
196
|
+
plan: expect.objectContaining({
|
|
197
|
+
sourceTableConfig: { orders: { mongodb_filter_expression: 'disabled' } }
|
|
198
|
+
})
|
|
199
|
+
})
|
|
200
|
+
}),
|
|
201
|
+
plan: expect.objectContaining({
|
|
202
|
+
plan: expect.objectContaining({
|
|
203
|
+
version: 3,
|
|
204
|
+
sourceTableConfig: { orders: { mongodb_filter_expression: 'disabled' } }
|
|
205
|
+
})
|
|
206
|
+
})
|
|
207
|
+
})
|
|
208
|
+
})
|
|
209
|
+
);
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
|
|
57
213
|
describe('AbstractReplicator heartbeat interval', () => {
|
|
58
214
|
const options: AbstractReplicatorOptions = {
|
|
59
215
|
id: 'test',
|
|
@@ -205,4 +205,132 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
205
205
|
expect(result!.version_label).toBe('v6');
|
|
206
206
|
expect(result!.errors).toEqual([expect.objectContaining({ level: 'fatal', message: 'Invalid sync config' })]);
|
|
207
207
|
});
|
|
208
|
+
|
|
209
|
+
test('reports each sync config error with its source location when parsing fails', async () => {
|
|
210
|
+
const yaml = `
|
|
211
|
+
config:
|
|
212
|
+
edition: 3
|
|
213
|
+
streams:
|
|
214
|
+
global:
|
|
215
|
+
query: SELECT id FROM test_table
|
|
216
|
+
unknown_key: true
|
|
217
|
+
`;
|
|
218
|
+
const content = makeSyncRulesContent();
|
|
219
|
+
content.parsed = (options?: any) => {
|
|
220
|
+
SqlSyncRules.fromYaml(yaml, { ...options, defaultSchema: 'public', throwOnError: true });
|
|
221
|
+
throw new Error('Expected parsing to fail');
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
const result = await getSyncRulesStatus(makeRouteAPI(), content, OPTIONS, makeSystemStorage());
|
|
225
|
+
|
|
226
|
+
expect(result!.errors).toHaveLength(1);
|
|
227
|
+
const [error] = result!.errors;
|
|
228
|
+
expect(error.level).toBe('fatal');
|
|
229
|
+
expect(error.message).not.toContain('Expected parsing to fail');
|
|
230
|
+
expect(yaml.slice(error.location!.start_offset, error.location!.end_offset)).toBe('unknown_key');
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
describe('source capability validation', () => {
|
|
235
|
+
test('exposes stored replication errors when source checks cannot run', async () => {
|
|
236
|
+
const content = makeSyncRulesContent({
|
|
237
|
+
status: {
|
|
238
|
+
id: '1',
|
|
239
|
+
replicationStreamId: 1,
|
|
240
|
+
state: storage.SyncRuleState.PROCESSING,
|
|
241
|
+
last_checkpoint_lsn: null,
|
|
242
|
+
last_fatal_error: 'Source capability unavailable',
|
|
243
|
+
last_fatal_error_ts: new Date(),
|
|
244
|
+
last_keepalive_ts: null,
|
|
245
|
+
last_checkpoint_ts: null
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
const result = await getSyncRulesStatus(
|
|
249
|
+
makeRouteAPI(),
|
|
250
|
+
content,
|
|
251
|
+
{
|
|
252
|
+
...OPTIONS,
|
|
253
|
+
check_connection: false,
|
|
254
|
+
active: false,
|
|
255
|
+
include_content: true
|
|
256
|
+
},
|
|
257
|
+
makeSystemStorage()
|
|
258
|
+
);
|
|
259
|
+
expect(result?.content).toBe(MINIMAL_SYNC_RULES);
|
|
260
|
+
expect(result?.errors).toContainEqual(
|
|
261
|
+
expect.objectContaining({
|
|
262
|
+
level: 'fatal',
|
|
263
|
+
message: 'Source capability unavailable'
|
|
264
|
+
})
|
|
265
|
+
);
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
test('reports capability failures alongside table diagnostics', async () => {
|
|
269
|
+
const api = makeRouteAPI();
|
|
270
|
+
api.validateSourceCapabilities = async () => [
|
|
271
|
+
{ level: 'fatal', message: 'Source capability unavailable' },
|
|
272
|
+
{ level: 'warning', message: 'Source advisory', location: { start_offset: 1, end_offset: 4 } }
|
|
273
|
+
];
|
|
274
|
+
api.getDebugTablesInfo = async () => [
|
|
275
|
+
{
|
|
276
|
+
schema: 'public',
|
|
277
|
+
pattern: 'items',
|
|
278
|
+
wildcard: false,
|
|
279
|
+
table: {
|
|
280
|
+
schema: 'public',
|
|
281
|
+
name: 'items',
|
|
282
|
+
data_queries: true,
|
|
283
|
+
parameter_queries: false,
|
|
284
|
+
replication_id: [],
|
|
285
|
+
errors: [{ level: 'warning', message: 'Index missing' }]
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
];
|
|
289
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
290
|
+
expect(result!.errors).toEqual(
|
|
291
|
+
expect.arrayContaining([
|
|
292
|
+
expect.objectContaining({ level: 'fatal', message: 'Source capability unavailable' }),
|
|
293
|
+
expect.objectContaining({ level: 'warning', message: 'Index missing' }),
|
|
294
|
+
expect.objectContaining({
|
|
295
|
+
level: 'warning',
|
|
296
|
+
message: 'Source advisory',
|
|
297
|
+
location: { start_offset: 1, end_offset: 4 }
|
|
298
|
+
})
|
|
299
|
+
])
|
|
300
|
+
);
|
|
301
|
+
expect(result!.connections[0].tables).toHaveLength(1);
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
test('preserves both validation failures and continues independent diagnostics', async () => {
|
|
305
|
+
const api = makeRouteAPI({
|
|
306
|
+
wal_status: 'extended',
|
|
307
|
+
safe_wal_size: 4 * GB,
|
|
308
|
+
max_slot_wal_keep_size: 10 * GB
|
|
309
|
+
});
|
|
310
|
+
api.validateSourceCapabilities = async () => {
|
|
311
|
+
throw new Error('Source capability unavailable');
|
|
312
|
+
};
|
|
313
|
+
api.getDebugTablesInfo = async () => {
|
|
314
|
+
throw new Error('Table metadata unavailable');
|
|
315
|
+
};
|
|
316
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
317
|
+
expect(result!.errors).toEqual(
|
|
318
|
+
expect.arrayContaining([
|
|
319
|
+
expect.objectContaining({ level: 'fatal', message: 'Source capability unavailable' }),
|
|
320
|
+
expect.objectContaining({ level: 'fatal', message: 'Table metadata unavailable' }),
|
|
321
|
+
expect.objectContaining({ level: 'warning', message: expect.stringContaining('WAL budget') })
|
|
322
|
+
])
|
|
323
|
+
);
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
test('does not check capabilities when connection checks are disabled', async () => {
|
|
327
|
+
const api = makeRouteAPI();
|
|
328
|
+
let called = false;
|
|
329
|
+
api.validateSourceCapabilities = async () => {
|
|
330
|
+
called = true;
|
|
331
|
+
return [];
|
|
332
|
+
};
|
|
333
|
+
await getSyncRulesStatus(api, makeSyncRulesContent(), { ...OPTIONS, check_connection: false }, makeSystemStorage());
|
|
334
|
+
expect(called).toBe(false);
|
|
335
|
+
});
|
|
208
336
|
});
|