@powersync/service-core 1.26.1 → 1.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/dist/api/RouteAPI.d.ts +8 -1
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.js +33 -16
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/source-capabilities.d.ts +7 -0
- package/dist/api/source-capabilities.js +23 -0
- package/dist/api/source-capabilities.js.map +1 -0
- package/dist/auth/JwtPayload.d.ts +2 -2
- package/dist/auth/JwtPayload.js +2 -2
- package/dist/auth/JwtPayload.js.map +1 -1
- package/dist/entry/commands/compact-action.js +1 -1
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/config-command.js +1 -1
- package/dist/entry/commands/config-command.js.map +1 -1
- package/dist/migrations/PowerSyncMigrationManager.js +2 -1
- package/dist/migrations/PowerSyncMigrationManager.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +4 -6
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +15 -0
- package/dist/replication/AbstractReplicator.js +37 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +0 -7
- package/dist/replication/ReplicationModule.js +13 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +39 -27
- package/dist/routes/endpoints/admin.d.ts +78 -54
- package/dist/routes/endpoints/admin.js +24 -10
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +7 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +39 -6
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +20 -3
- package/dist/storage/BucketStorageFactory.js +25 -2
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.d.ts +2 -1
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +3 -0
- package/dist/storage/StorageVersionConfig.d.ts +0 -4
- package/dist/storage/StorageVersionConfig.js +0 -4
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +3 -1
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +9 -5
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +1 -1
- package/dist/sync/BucketChecksumState.js +5 -4
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/util/param-logging.d.ts +5 -2
- package/dist/util/param-logging.js +17 -9
- package/dist/util/param-logging.js.map +1 -1
- package/dist/util/protocol-types.d.ts +7 -7
- package/dist/util/protocol-types.js +3 -3
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +9 -1
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +30 -14
- package/src/api/source-capabilities.ts +24 -0
- package/src/auth/JwtPayload.ts +2 -2
- package/src/entry/commands/compact-action.ts +1 -1
- package/src/entry/commands/config-command.ts +1 -1
- package/src/migrations/PowerSyncMigrationManager.ts +2 -1
- package/src/replication/AbstractReplicationJob.ts +2 -4
- package/src/replication/AbstractReplicator.ts +46 -5
- package/src/replication/ReplicationModule.ts +12 -12
- package/src/routes/endpoints/admin.ts +29 -11
- package/src/routes/endpoints/socket-route.ts +9 -6
- package/src/routes/endpoints/sync-rules.ts +42 -9
- package/src/storage/BucketStorageFactory.ts +33 -4
- package/src/storage/PersistedSyncConfigContent.ts +3 -0
- package/src/storage/StorageProvider.ts +3 -0
- package/src/storage/StorageVersionConfig.ts +0 -5
- package/src/storage/SyncRulesBucketStorage.ts +3 -1
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +17 -5
- package/src/sync/BucketChecksumState.ts +5 -4
- package/src/util/param-logging.ts +21 -14
- package/src/util/protocol-types.ts +5 -5
- package/test/src/AbstractReplicator.test.ts +156 -0
- package/test/src/broadcast_iterable.test.ts +26 -8
- package/test/src/diagnostics.test.ts +128 -0
- package/test/src/routes/admin.test.ts +235 -4
- package/test/src/routes/sync-rules.test.ts +124 -0
- package/test/src/source_table_config.test.ts +104 -0
- package/test/src/storage/BucketStorageFactory.test.ts +46 -0
- package/test/src/sync/BucketChecksumState.test.ts +195 -6
- package/test/src/util/param-logging.test.ts +59 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -16,6 +16,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
16
16
|
const { routerEngine, metricsEngine, syncContext } = service_context;
|
|
17
17
|
const streamStart = Date.now();
|
|
18
18
|
|
|
19
|
+
// Format client-controlled values before registering connection resources.
|
|
20
|
+
const formattedAppMetadata = params.app_metadata ? limitParamsForLogging(params.app_metadata) : undefined;
|
|
21
|
+
const formattedClientParams = params.parameters ? limitParamsForLogging(params.parameters) : undefined;
|
|
22
|
+
|
|
19
23
|
logger.defaultMeta = {
|
|
20
24
|
...logger.defaultMeta,
|
|
21
25
|
user_id: context.token_payload!.userIdJson,
|
|
@@ -99,13 +103,12 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
99
103
|
tracker.setCompressed(connection.tracker.encoding);
|
|
100
104
|
}
|
|
101
105
|
|
|
102
|
-
const formattedAppMetadata = params.app_metadata ? limitParamsForLogging(params.app_metadata) : undefined;
|
|
103
|
-
logger.info('Sync stream started', {
|
|
104
|
-
app_metadata: formattedAppMetadata,
|
|
105
|
-
client_params: params.parameters ? limitParamsForLogging(params.parameters) : undefined
|
|
106
|
-
});
|
|
107
|
-
|
|
108
106
|
try {
|
|
107
|
+
logger.info('Sync stream started', {
|
|
108
|
+
app_metadata: formattedAppMetadata,
|
|
109
|
+
client_params: formattedClientParams
|
|
110
|
+
});
|
|
111
|
+
|
|
109
112
|
for await (const data of sync.streamResponse({
|
|
110
113
|
syncContext: syncContext,
|
|
111
114
|
bucketStorage: bucketStorage,
|
|
@@ -2,9 +2,10 @@ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-frame
|
|
|
2
2
|
import { SqlSyncRules, SyncConfigWithErrors, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
3
3
|
import type { FastifyPluginAsync } from 'fastify';
|
|
4
4
|
import * as t from 'ts-codec';
|
|
5
|
+
import { assertSourceCapabilities } from '../../api/source-capabilities.js';
|
|
5
6
|
|
|
6
|
-
import { RouteAPI } from '../../api/RouteAPI.js';
|
|
7
|
-
import {
|
|
7
|
+
import { PatternResult, RouteAPI } from '../../api/RouteAPI.js';
|
|
8
|
+
import { logSyncConfigErrors, updateSyncRulesFromConfig } from '../../storage/BucketStorageFactory.js';
|
|
8
9
|
import { authApi } from '../auth.js';
|
|
9
10
|
import { routeDefinition } from '../router.js';
|
|
10
11
|
|
|
@@ -55,6 +56,7 @@ export const deploySyncRules = routeDefinition({
|
|
|
55
56
|
let syncConfig: SyncConfigWithErrors;
|
|
56
57
|
|
|
57
58
|
try {
|
|
59
|
+
// First ensure the SyncConfig parses correctly
|
|
58
60
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
59
61
|
syncConfig = SqlSyncRules.fromYaml(content, {
|
|
60
62
|
...apiHandler.getParseSyncRulesOptions(),
|
|
@@ -70,6 +72,10 @@ export const deploySyncRules = routeDefinition({
|
|
|
70
72
|
});
|
|
71
73
|
}
|
|
72
74
|
|
|
75
|
+
logSyncConfigErrors(syncConfig);
|
|
76
|
+
// Validate if the current sources support the configuration specified in the sync config
|
|
77
|
+
await assertSourceCapabilities(service_context.routerEngine.getAPI(), syncConfig.config);
|
|
78
|
+
|
|
73
79
|
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
|
|
74
80
|
|
|
75
81
|
return {
|
|
@@ -176,11 +182,18 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
176
182
|
}
|
|
177
183
|
|
|
178
184
|
const sync_rules = active.content;
|
|
185
|
+
const parsed = SqlSyncRules.fromYaml(sync_rules.sync_rules_content, {
|
|
186
|
+
...payload.context.service_context.routerEngine.getAPI().getParseSyncRulesOptions(),
|
|
187
|
+
schema: undefined,
|
|
188
|
+
// This sync config already passed validation. But if the rules are not valid anymore due
|
|
189
|
+
// to a service change, we do want to report the error here.
|
|
190
|
+
throwOnError: true
|
|
191
|
+
});
|
|
192
|
+
logSyncConfigErrors(parsed);
|
|
193
|
+
await assertSourceCapabilities(payload.context.service_context.routerEngine.getAPI(), parsed.config);
|
|
194
|
+
|
|
179
195
|
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
180
|
-
|
|
181
|
-
// This sync config already passed validation. But if the rules are not valid anymore due
|
|
182
|
-
// to a service change, we do want to report the error here.
|
|
183
|
-
validate: true,
|
|
196
|
+
updateSyncRulesFromConfig(parsed, {
|
|
184
197
|
version_label: sync_rules.version_label,
|
|
185
198
|
forceNewReplicationStream: true
|
|
186
199
|
})
|
|
@@ -208,11 +221,31 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
|
|
|
208
221
|
// No schema-based validation at this point
|
|
209
222
|
schema: undefined
|
|
210
223
|
});
|
|
211
|
-
const
|
|
212
|
-
const
|
|
224
|
+
const errors: string[] = [];
|
|
225
|
+
const warnings = rules.errors.filter((error) => error.type === 'warning').map((error) => error.message);
|
|
226
|
+
const recordError = (error: any) => {
|
|
227
|
+
errors.push(...(error instanceof SyncRulesErrors ? error.errors.map((e) => e.message) : [error.message]));
|
|
228
|
+
};
|
|
229
|
+
try {
|
|
230
|
+
const diagnostics = (await apiHandler.validateSourceCapabilities?.(rules.config)) ?? [];
|
|
231
|
+
for (const diagnostic of diagnostics) {
|
|
232
|
+
(diagnostic.level === 'fatal' ? errors : warnings).push(diagnostic.message);
|
|
233
|
+
}
|
|
234
|
+
} catch (e) {
|
|
235
|
+
recordError(e);
|
|
236
|
+
}
|
|
237
|
+
let resolved_tables: PatternResult[] = [];
|
|
238
|
+
try {
|
|
239
|
+
const source_table_patterns = rules.config.getSourceTables();
|
|
240
|
+
resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);
|
|
241
|
+
} catch (e) {
|
|
242
|
+
recordError(e);
|
|
243
|
+
}
|
|
213
244
|
|
|
214
245
|
return {
|
|
215
|
-
valid:
|
|
246
|
+
valid: errors.length == 0,
|
|
247
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
248
|
+
...(errors.length > 0 ? { errors } : {}),
|
|
216
249
|
bucket_definitions: rules.config.debugRepresentation(),
|
|
217
250
|
source_tables: resolved_tables,
|
|
218
251
|
data_tables: rules.config.debugGetOutputTables()
|
|
@@ -117,10 +117,14 @@ export abstract class BucketStorageFactory
|
|
|
117
117
|
*/
|
|
118
118
|
abstract getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
async [Symbol.asyncDispose](): Promise<void> {
|
|
121
|
+
await this.iterateAsyncListeners(async (listener) => listener.beforeDispose?.());
|
|
122
|
+
}
|
|
121
123
|
}
|
|
122
124
|
|
|
123
125
|
export interface BucketStorageFactoryListener {
|
|
126
|
+
/** Release owned resources before the factory closes its database connections. */
|
|
127
|
+
beforeDispose: () => Promise<void>;
|
|
124
128
|
syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
|
|
125
129
|
replicationEvent: (event: ReplicationEventPayload) => void;
|
|
126
130
|
}
|
|
@@ -177,7 +181,7 @@ export interface UpdateSyncRulesOptions {
|
|
|
177
181
|
config: {
|
|
178
182
|
yaml: string;
|
|
179
183
|
/**
|
|
180
|
-
* The serialized sync plan for the sync configuration, or `null` for configurations not using the
|
|
184
|
+
* The serialized sync plan for the sync configuration, or `null` for configurations not using the Sync Streams
|
|
181
185
|
* compiler.
|
|
182
186
|
*/
|
|
183
187
|
plan: SerializedSyncPlan | null;
|
|
@@ -223,14 +227,15 @@ export function updateSyncRulesFromYaml(
|
|
|
223
227
|
content: string,
|
|
224
228
|
options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
|
|
225
229
|
): UpdateSyncRulesOptions {
|
|
230
|
+
const { validate, ...updateOptions } = options ?? {};
|
|
226
231
|
const config = SqlSyncRules.fromYaml(content, {
|
|
227
232
|
// No schema-based validation at this point
|
|
228
233
|
schema: undefined,
|
|
229
234
|
defaultSchema: options?.defaultSchema ?? 'not_applicable', // Not needed for validation
|
|
230
|
-
throwOnError:
|
|
235
|
+
throwOnError: validate ?? false
|
|
231
236
|
});
|
|
232
237
|
|
|
233
|
-
return updateSyncRulesFromConfig(config,
|
|
238
|
+
return updateSyncRulesFromConfig(config, updateOptions);
|
|
234
239
|
}
|
|
235
240
|
|
|
236
241
|
export function updateSyncRulesFromConfig(
|
|
@@ -257,7 +262,31 @@ export function updateSyncRulesFromConfig(
|
|
|
257
262
|
return { config: { yaml: config.content, plan, parsed }, ...options };
|
|
258
263
|
}
|
|
259
264
|
|
|
265
|
+
/**
|
|
266
|
+
* Logs the errors and warnings of a parsed sync config
|
|
267
|
+
*
|
|
268
|
+
* Call this where a sync config is loaded or deployed, so that operators see these in the service logs and not only
|
|
269
|
+
* in the diagnostics and validate API responses.
|
|
270
|
+
*
|
|
271
|
+
* Fatal errors usually throw during parsing before this is reached. They can reach this when the sync config is parsed
|
|
272
|
+
* without validation, such as at startup with `exit_on_error: false`.
|
|
273
|
+
*/
|
|
274
|
+
export function logSyncConfigErrors(parsed: SyncConfigWithErrors, log: typeof logger = logger) {
|
|
275
|
+
for (const error of parsed.errors) {
|
|
276
|
+
if (error.type == 'warning') {
|
|
277
|
+
log.warn(`Sync config warning: ${error.message}`);
|
|
278
|
+
} else {
|
|
279
|
+
log.error(`Sync config error: ${error.message}`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
260
284
|
export interface GetIntanceOptions {
|
|
285
|
+
/**
|
|
286
|
+
* The job lease, including a lease acquired during initial configuration.
|
|
287
|
+
* Required for writing; optional for reading.
|
|
288
|
+
*/
|
|
289
|
+
replicationLock?: ReplicationLock;
|
|
261
290
|
/**
|
|
262
291
|
* Set to true to skip trigger any events for creating the instance.
|
|
263
292
|
*
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
HydrationState,
|
|
11
11
|
nodeSqlite,
|
|
12
12
|
PrecompiledSyncConfig,
|
|
13
|
+
SourceConnectionMap,
|
|
13
14
|
SqlSyncRules,
|
|
14
15
|
SyncConfigWithErrors,
|
|
15
16
|
versionedHydrationState,
|
|
@@ -202,6 +203,8 @@ export interface PersistedSyncConfigContentData {
|
|
|
202
203
|
readonly version_label?: string;
|
|
203
204
|
}
|
|
204
205
|
export type PersistedSyncConfigId = string;
|
|
206
|
+
|
|
205
207
|
export interface ParseSyncConfigOptions {
|
|
206
208
|
defaultSchema: string;
|
|
209
|
+
connections?: SourceConnectionMap;
|
|
207
210
|
}
|
|
@@ -49,11 +49,6 @@ export const STORAGE_VERSION_4 = 4;
|
|
|
49
49
|
*/
|
|
50
50
|
export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
|
|
51
51
|
|
|
52
|
-
/**
|
|
53
|
-
* Default storage version for newly persisted replication streams.
|
|
54
|
-
*/
|
|
55
|
-
export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
|
|
56
|
-
|
|
57
52
|
/**
|
|
58
53
|
* All versions that can be loaded.
|
|
59
54
|
*
|
|
@@ -11,7 +11,7 @@ import * as util from '../util/util-index.js';
|
|
|
11
11
|
import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
|
|
12
12
|
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
13
13
|
import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
|
|
14
|
-
import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
|
|
14
|
+
import type { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
|
|
15
15
|
import { SourceEntityDescriptor } from './SourceEntity.js';
|
|
16
16
|
import { SourceTable } from './SourceTable.js';
|
|
17
17
|
import { SourceTableCandidateReconciler } from './SourceTableReconciler.js';
|
|
@@ -402,6 +402,8 @@ export interface CompactOptions {
|
|
|
402
402
|
}
|
|
403
403
|
|
|
404
404
|
export interface CompactInitialReplicationOptions {
|
|
405
|
+
/** Internal/testing use: bypass the new-chunk threshold in MongoDB V3/V4 storage. */
|
|
406
|
+
forceChunkCompaction?: boolean;
|
|
405
407
|
/**
|
|
406
408
|
* Compact data up to this op id.
|
|
407
409
|
*
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CompatibilityContext } from '@powersync/service-sync-rules';
|
|
1
|
+
import { CompatibilityContext, sourceTableConfigsEqual } from '@powersync/service-sync-rules';
|
|
2
2
|
import { Logger } from 'winston';
|
|
3
3
|
import { SerializedSyncPlan, UpdateSyncRulesOptions } from '../BucketStorageFactory.js';
|
|
4
4
|
|
|
@@ -11,8 +11,8 @@ export function isCompatible(
|
|
|
11
11
|
logger: Logger
|
|
12
12
|
): boolean {
|
|
13
13
|
if (updateConfig.plan == null) {
|
|
14
|
-
// Only support
|
|
15
|
-
logger.info(`Not using current
|
|
14
|
+
// Only support Sync Streams with serialized plans
|
|
15
|
+
logger.info(`Not using current Sync Streams - incremental reprocessing not supported`);
|
|
16
16
|
return false;
|
|
17
17
|
}
|
|
18
18
|
|
|
@@ -23,8 +23,8 @@ export function isCompatible(
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
if (existingPlans.some((plan) => plan == null)) {
|
|
26
|
-
// Only support
|
|
27
|
-
logger.info(`Existing replication stream not using current
|
|
26
|
+
// Only support Sync Streams with serialized plans
|
|
27
|
+
logger.info(`Existing replication stream not using current Sync Streams - incremental reprocessing not supported`);
|
|
28
28
|
return false;
|
|
29
29
|
}
|
|
30
30
|
|
|
@@ -37,5 +37,17 @@ export function isCompatible(
|
|
|
37
37
|
return false;
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
if (
|
|
41
|
+
existingPlans.some(
|
|
42
|
+
(existing) =>
|
|
43
|
+
!sourceTableConfigsEqual(
|
|
44
|
+
existing!.plan.sourceTableConfig ?? {},
|
|
45
|
+
updateConfig.plan!.plan.sourceTableConfig ?? {}
|
|
46
|
+
)
|
|
47
|
+
)
|
|
48
|
+
) {
|
|
49
|
+
logger.info('Source table configuration changed - incremental reprocessing not supported');
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
40
52
|
return true;
|
|
41
53
|
}
|
|
@@ -89,7 +89,7 @@ export class BucketChecksumState {
|
|
|
89
89
|
* subscribed to, we keep an index of the stream names to their index in that array.
|
|
90
90
|
*
|
|
91
91
|
* This is used to compress the representation of buckets in `checkpoint` and `checkpoint_diff` lines: For buckets
|
|
92
|
-
* that are part of
|
|
92
|
+
* that are part of legacy Sync Rules or default streams, we need to include the name of the defining bucket definition or stream
|
|
93
93
|
* yielding that bucket (so that clients can track progress for default streams).
|
|
94
94
|
* But instead of sending the name for each bucket, we use the fact that it's part of the streams array and only send
|
|
95
95
|
* their index, reducing the size of those messages.
|
|
@@ -747,7 +747,8 @@ export class BucketParameterState {
|
|
|
747
747
|
}
|
|
748
748
|
}
|
|
749
749
|
}
|
|
750
|
-
|
|
750
|
+
// A stream can reach the same bucket through both static and dynamic parameters.
|
|
751
|
+
const allBuckets = mergeBuckets([...staticBuckets, ...dynamicBuckets]);
|
|
751
752
|
|
|
752
753
|
if (invalidateDataBuckets) {
|
|
753
754
|
return {
|
|
@@ -816,7 +817,7 @@ function logCheckpoint(
|
|
|
816
817
|
}
|
|
817
818
|
|
|
818
819
|
/**
|
|
819
|
-
* Format a breakdown of dynamic buckets by
|
|
820
|
+
* Format a breakdown of dynamic buckets by Sync Stream or legacy bucket definition.
|
|
820
821
|
*
|
|
821
822
|
* Sorts definitions by count (descending), includes the top 100, and returns both the
|
|
822
823
|
* formatted message string and the counts record suitable for structured log data.
|
|
@@ -834,7 +835,7 @@ function formatBucketDefinitionBreakdown(
|
|
|
834
835
|
const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
|
|
835
836
|
const sortedDefinitions = allSorted.slice(0, maxLoggedDefinitions);
|
|
836
837
|
|
|
837
|
-
const sourceLabel = bucketSourceType == BucketSourceType.SYNC_STREAM ? '
|
|
838
|
+
const sourceLabel = bucketSourceType == BucketSourceType.SYNC_STREAM ? 'Sync Stream' : 'bucket definition';
|
|
838
839
|
let message = `\nBuckets by ${sourceLabel}:`;
|
|
839
840
|
const countsByDefinition: Record<string, number> = {};
|
|
840
841
|
for (const [definition, count] of sortedDefinitions) {
|
|
@@ -18,8 +18,9 @@ export const DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS: ParamLoggingFormatOptions = {
|
|
|
18
18
|
* Formats potentially arbitrary parameters for logging.
|
|
19
19
|
* This limits the number of keys and strings to a maximum length.
|
|
20
20
|
* A warning key-value is added if the number of keys exceeds the maximum.
|
|
21
|
-
*
|
|
21
|
+
* Keys and string values exceeding the maximum length are truncated.
|
|
22
22
|
* Non-String values are stringified, the maximum length is then applied.
|
|
23
|
+
* Values that cannot be serialized are replaced with a placeholder.
|
|
23
24
|
* @param params - The parameters to format.
|
|
24
25
|
* @param options - The options to use.
|
|
25
26
|
* @default DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS
|
|
@@ -41,20 +42,26 @@ export function limitParamsForLogging(
|
|
|
41
42
|
return value;
|
|
42
43
|
}
|
|
43
44
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
function formatValue(value: unknown): string {
|
|
46
|
+
if (typeof value == 'string') {
|
|
47
|
+
return trimString(value);
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
return trimString(JSON.stringify(value) ?? '[undefined]');
|
|
51
|
+
} catch {
|
|
52
|
+
return trimString('[unserializable]');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
49
55
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
56
|
+
return Object.fromEntries(
|
|
57
|
+
Object.entries(params)
|
|
58
|
+
.slice(0, maxKeyCount + 1)
|
|
59
|
+
.map(([key, value], index) => {
|
|
60
|
+
if (index == maxKeyCount) {
|
|
61
|
+
return ['⚠️', 'Additional parameters omitted'];
|
|
62
|
+
}
|
|
53
63
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
return [key, trimString(JSON.stringify(value))];
|
|
58
|
-
})
|
|
64
|
+
return [trimString(key), formatValue(value)];
|
|
65
|
+
})
|
|
59
66
|
);
|
|
60
67
|
}
|
|
@@ -19,7 +19,7 @@ export type BucketRequest = t.Decoded<typeof BucketRequest>;
|
|
|
19
19
|
*/
|
|
20
20
|
export const RequestedStreamSubscription = t.object({
|
|
21
21
|
/**
|
|
22
|
-
* The defined name of the stream as it appears in
|
|
22
|
+
* The defined name of the stream as it appears in Sync Stream definitions.
|
|
23
23
|
*/
|
|
24
24
|
stream: t.string,
|
|
25
25
|
/**
|
|
@@ -29,7 +29,7 @@ export const RequestedStreamSubscription = t.object({
|
|
|
29
29
|
/**
|
|
30
30
|
* Set when the client wishes to re-assign a different priority to this stream.
|
|
31
31
|
*
|
|
32
|
-
* Streams and
|
|
32
|
+
* Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
|
|
33
33
|
* useful when the priority for partial syncs depends on e.g. the current page opened in a client.
|
|
34
34
|
*/
|
|
35
35
|
override_priority: orNull(t.number)
|
|
@@ -49,7 +49,7 @@ export const StreamSubscriptionRequest = t.object({
|
|
|
49
49
|
include_defaults: t.boolean.optional(),
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
|
-
* An array of
|
|
52
|
+
* An array of Sync Streams the client has opened explicitly.
|
|
53
53
|
*/
|
|
54
54
|
subscriptions: t.array(RequestedStreamSubscription)
|
|
55
55
|
});
|
|
@@ -253,8 +253,8 @@ export interface BucketChecksum {
|
|
|
253
253
|
/**
|
|
254
254
|
* The reason a particular bucket is included in a checkpoint.
|
|
255
255
|
*
|
|
256
|
-
* This information allows clients to associate individual buckets with
|
|
257
|
-
* association is useful because it enables clients to track progress for individual
|
|
256
|
+
* This information allows clients to associate individual buckets with Sync Streams they're subscribed to. Having that
|
|
257
|
+
* association is useful because it enables clients to track progress for individual Sync Streams.
|
|
258
258
|
*/
|
|
259
259
|
export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
|
|
260
260
|
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import type { RouteAPI } from '@/api/RouteAPI.js';
|
|
2
|
+
import { assertSourceCapabilities } from '@/api/source-capabilities.js';
|
|
1
3
|
import { AbstractReplicationJob } from '@/replication/AbstractReplicationJob.js';
|
|
2
4
|
import { AbstractReplicator, AbstractReplicatorOptions, CreateJobOptions } from '@/replication/AbstractReplicator.js';
|
|
3
5
|
import { PersistedReplicationStream } from '@/storage/PersistedReplicationStream.js';
|
|
6
|
+
import type { ReplicationLock } from '@/storage/ReplicationLock.js';
|
|
4
7
|
import { SyncRulesBucketStorage } from '@/storage/SyncRulesBucketStorage.js';
|
|
5
8
|
import { describe, expect, it, vi } from 'vitest';
|
|
6
9
|
|
|
@@ -45,6 +48,21 @@ class TestReplicator extends AbstractReplicator {
|
|
|
45
48
|
return (this as any).heartbeatIntervalNanos;
|
|
46
49
|
}
|
|
47
50
|
|
|
51
|
+
async runStartupForTest(): Promise<void> {
|
|
52
|
+
const controller = new AbortController();
|
|
53
|
+
controller.abort();
|
|
54
|
+
(this as any).abortController = controller;
|
|
55
|
+
await (this as any).runLoop();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async refreshForTest(configuredLock?: ReplicationLock): Promise<{ replicationJobStarted: boolean }> {
|
|
59
|
+
(this as any).abortController = new AbortController();
|
|
60
|
+
return (this as any).refresh({
|
|
61
|
+
configuredLock,
|
|
62
|
+
loadedVersionLabel: this.syncRuleProvider.versionLabel
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
|
|
48
66
|
shouldHandleStreamForTest(
|
|
49
67
|
replicationStream: PersistedReplicationStream,
|
|
50
68
|
loadedSyncRules: string | undefined,
|
|
@@ -54,6 +72,144 @@ class TestReplicator extends AbstractReplicator {
|
|
|
54
72
|
}
|
|
55
73
|
}
|
|
56
74
|
|
|
75
|
+
describe('AbstractReplicator startup sync config', () => {
|
|
76
|
+
it.each([true, false])('respects exit_on_error=%s for unsupported file-loaded filters', async (exitOnError) => {
|
|
77
|
+
const yaml = /* yaml */ `
|
|
78
|
+
# sync config fixture.
|
|
79
|
+
config:
|
|
80
|
+
edition: 3
|
|
81
|
+
source_table_options:
|
|
82
|
+
orders:
|
|
83
|
+
mongodb_filter_expression: { $eq: ['$$doc.active', true] }
|
|
84
|
+
streams:
|
|
85
|
+
orders:
|
|
86
|
+
query: SELECT * FROM orders
|
|
87
|
+
`;
|
|
88
|
+
const configureSyncRules = vi.fn(
|
|
89
|
+
async (_options: import('@/storage/BucketStorageFactory.js').UpdateSyncRulesOptions) => ({ updated: false })
|
|
90
|
+
);
|
|
91
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
92
|
+
id: 'test',
|
|
93
|
+
storageEngine: {
|
|
94
|
+
activeBucketStorage: { configureSyncRules }
|
|
95
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
96
|
+
configuration: { connections: [{ type: 'postgresql' }] },
|
|
97
|
+
syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: undefined },
|
|
98
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
99
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
100
|
+
});
|
|
101
|
+
if (exitOnError) {
|
|
102
|
+
await expect(replicator.runStartupForTest()).rejects.toThrow('only supported by MongoDB sources');
|
|
103
|
+
expect(configureSyncRules).not.toHaveBeenCalled();
|
|
104
|
+
} else {
|
|
105
|
+
await expect(replicator.runStartupForTest()).resolves.toBeUndefined();
|
|
106
|
+
expect(configureSyncRules).toHaveBeenCalledOnce();
|
|
107
|
+
expect(configureSyncRules.mock.calls[0][0].config.plan?.errors).toEqual([
|
|
108
|
+
expect.objectContaining({
|
|
109
|
+
level: 'fatal',
|
|
110
|
+
message: expect.stringContaining('only supported by MongoDB sources')
|
|
111
|
+
})
|
|
112
|
+
]);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it.each(
|
|
117
|
+
[true, false].flatMap((exitOnError) =>
|
|
118
|
+
(['warning', 'valid', 'unexpected error'] as const).map((result) => ({ exitOnError, result }))
|
|
119
|
+
)
|
|
120
|
+
)(
|
|
121
|
+
'checks source capabilities before persistence: $result, exit_on_error=$exitOnError',
|
|
122
|
+
async ({ result, exitOnError }) => {
|
|
123
|
+
const yaml = /* yaml */ `
|
|
124
|
+
# sync config fixture.
|
|
125
|
+
config:
|
|
126
|
+
edition: 3
|
|
127
|
+
streams:
|
|
128
|
+
orders:
|
|
129
|
+
query: SELECT * FROM orders
|
|
130
|
+
`;
|
|
131
|
+
const calls: string[] = [];
|
|
132
|
+
const configureSyncRules = vi.fn(async () => {
|
|
133
|
+
calls.push('persist');
|
|
134
|
+
return { updated: false };
|
|
135
|
+
});
|
|
136
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
137
|
+
id: 'test',
|
|
138
|
+
storageEngine: {
|
|
139
|
+
activeBucketStorage: { configureSyncRules }
|
|
140
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
141
|
+
syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: undefined },
|
|
142
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
143
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
144
|
+
});
|
|
145
|
+
replicator.registerSourceCapabilitiesAssertion((config) =>
|
|
146
|
+
assertSourceCapabilities(
|
|
147
|
+
{
|
|
148
|
+
async validateSourceCapabilities() {
|
|
149
|
+
calls.push('validate');
|
|
150
|
+
if (result === 'unexpected error') throw new Error('Source validation failed');
|
|
151
|
+
return result === 'warning' ? [{ level: 'warning', message: 'Missing index' }] : [];
|
|
152
|
+
}
|
|
153
|
+
} as unknown as RouteAPI,
|
|
154
|
+
config
|
|
155
|
+
)
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
if (result === 'unexpected error' && exitOnError) {
|
|
159
|
+
await expect(replicator.runStartupForTest()).rejects.toThrow('Source validation failed');
|
|
160
|
+
expect(calls).toEqual(['validate']);
|
|
161
|
+
} else {
|
|
162
|
+
await replicator.runStartupForTest();
|
|
163
|
+
expect(calls).toEqual(['validate', 'persist']);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
it.each([true, false])('retains source-table options with exit_on_error=%s', async (exitOnError) => {
|
|
169
|
+
const yaml = /* yaml */ `
|
|
170
|
+
{
|
|
171
|
+
config: { edition: 3, source_table_options: { orders: { mongodb_filter_expression: 'disabled' } } },
|
|
172
|
+
streams: { orders: { query: SELECT * FROM orders } }
|
|
173
|
+
}
|
|
174
|
+
`;
|
|
175
|
+
const configureSyncRules = vi.fn(async () => ({ updated: false }));
|
|
176
|
+
const replicator = new TestReplicator(async () => {}, {
|
|
177
|
+
id: 'test',
|
|
178
|
+
storageEngine: {
|
|
179
|
+
activeBucketStorage: { configureSyncRules }
|
|
180
|
+
} as unknown as AbstractReplicatorOptions['storageEngine'],
|
|
181
|
+
syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: 'v2' },
|
|
182
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
183
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
await replicator.runStartupForTest();
|
|
187
|
+
|
|
188
|
+
expect(configureSyncRules).toHaveBeenCalledExactlyOnceWith(
|
|
189
|
+
expect.objectContaining({
|
|
190
|
+
lock: true,
|
|
191
|
+
version_label: 'v2',
|
|
192
|
+
config: expect.objectContaining({
|
|
193
|
+
parsed: expect.objectContaining({
|
|
194
|
+
errors: [],
|
|
195
|
+
config: expect.objectContaining({
|
|
196
|
+
plan: expect.objectContaining({
|
|
197
|
+
sourceTableConfig: { orders: { mongodb_filter_expression: 'disabled' } }
|
|
198
|
+
})
|
|
199
|
+
})
|
|
200
|
+
}),
|
|
201
|
+
plan: expect.objectContaining({
|
|
202
|
+
plan: expect.objectContaining({
|
|
203
|
+
version: 3,
|
|
204
|
+
sourceTableConfig: { orders: { mongodb_filter_expression: 'disabled' } }
|
|
205
|
+
})
|
|
206
|
+
})
|
|
207
|
+
})
|
|
208
|
+
})
|
|
209
|
+
);
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
|
|
57
213
|
describe('AbstractReplicator heartbeat interval', () => {
|
|
58
214
|
const options: AbstractReplicatorOptions = {
|
|
59
215
|
id: 'test',
|