@powersync/service-core 1.26.1 → 1.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +67 -0
  2. package/dist/api/RouteAPI.d.ts +8 -1
  3. package/dist/api/api-index.d.ts +1 -0
  4. package/dist/api/api-index.js +1 -0
  5. package/dist/api/api-index.js.map +1 -1
  6. package/dist/api/diagnostics.js +33 -16
  7. package/dist/api/diagnostics.js.map +1 -1
  8. package/dist/api/source-capabilities.d.ts +7 -0
  9. package/dist/api/source-capabilities.js +23 -0
  10. package/dist/api/source-capabilities.js.map +1 -0
  11. package/dist/auth/JwtPayload.d.ts +2 -2
  12. package/dist/auth/JwtPayload.js +2 -2
  13. package/dist/auth/JwtPayload.js.map +1 -1
  14. package/dist/entry/commands/compact-action.js +1 -1
  15. package/dist/entry/commands/compact-action.js.map +1 -1
  16. package/dist/entry/commands/config-command.js +1 -1
  17. package/dist/entry/commands/config-command.js.map +1 -1
  18. package/dist/migrations/PowerSyncMigrationManager.js +2 -1
  19. package/dist/migrations/PowerSyncMigrationManager.js.map +1 -1
  20. package/dist/replication/AbstractReplicationJob.d.ts +4 -6
  21. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  22. package/dist/replication/AbstractReplicator.d.ts +15 -0
  23. package/dist/replication/AbstractReplicator.js +37 -5
  24. package/dist/replication/AbstractReplicator.js.map +1 -1
  25. package/dist/replication/ReplicationModule.d.ts +0 -7
  26. package/dist/replication/ReplicationModule.js +13 -3
  27. package/dist/replication/ReplicationModule.js.map +1 -1
  28. package/dist/routes/configure-fastify.d.ts +39 -27
  29. package/dist/routes/endpoints/admin.d.ts +78 -54
  30. package/dist/routes/endpoints/admin.js +24 -10
  31. package/dist/routes/endpoints/admin.js.map +1 -1
  32. package/dist/routes/endpoints/socket-route.js +7 -5
  33. package/dist/routes/endpoints/socket-route.js.map +1 -1
  34. package/dist/routes/endpoints/sync-rules.js +39 -6
  35. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  36. package/dist/storage/BucketStorageFactory.d.ts +20 -3
  37. package/dist/storage/BucketStorageFactory.js +25 -2
  38. package/dist/storage/BucketStorageFactory.js.map +1 -1
  39. package/dist/storage/PersistedSyncConfigContent.d.ts +2 -1
  40. package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
  41. package/dist/storage/StorageProvider.d.ts +3 -0
  42. package/dist/storage/StorageVersionConfig.d.ts +0 -4
  43. package/dist/storage/StorageVersionConfig.js +0 -4
  44. package/dist/storage/StorageVersionConfig.js.map +1 -1
  45. package/dist/storage/SyncRulesBucketStorage.d.ts +3 -1
  46. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  47. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +9 -5
  48. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -1
  49. package/dist/sync/BucketChecksumState.d.ts +1 -1
  50. package/dist/sync/BucketChecksumState.js +5 -4
  51. package/dist/sync/BucketChecksumState.js.map +1 -1
  52. package/dist/util/param-logging.d.ts +5 -2
  53. package/dist/util/param-logging.js +17 -9
  54. package/dist/util/param-logging.js.map +1 -1
  55. package/dist/util/protocol-types.d.ts +7 -7
  56. package/dist/util/protocol-types.js +3 -3
  57. package/package.json +11 -11
  58. package/src/api/RouteAPI.ts +9 -1
  59. package/src/api/api-index.ts +1 -0
  60. package/src/api/diagnostics.ts +30 -14
  61. package/src/api/source-capabilities.ts +24 -0
  62. package/src/auth/JwtPayload.ts +2 -2
  63. package/src/entry/commands/compact-action.ts +1 -1
  64. package/src/entry/commands/config-command.ts +1 -1
  65. package/src/migrations/PowerSyncMigrationManager.ts +2 -1
  66. package/src/replication/AbstractReplicationJob.ts +2 -4
  67. package/src/replication/AbstractReplicator.ts +46 -5
  68. package/src/replication/ReplicationModule.ts +12 -12
  69. package/src/routes/endpoints/admin.ts +29 -11
  70. package/src/routes/endpoints/socket-route.ts +9 -6
  71. package/src/routes/endpoints/sync-rules.ts +42 -9
  72. package/src/storage/BucketStorageFactory.ts +33 -4
  73. package/src/storage/PersistedSyncConfigContent.ts +3 -0
  74. package/src/storage/StorageProvider.ts +3 -0
  75. package/src/storage/StorageVersionConfig.ts +0 -5
  76. package/src/storage/SyncRulesBucketStorage.ts +3 -1
  77. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +17 -5
  78. package/src/sync/BucketChecksumState.ts +5 -4
  79. package/src/util/param-logging.ts +21 -14
  80. package/src/util/protocol-types.ts +5 -5
  81. package/test/src/AbstractReplicator.test.ts +156 -0
  82. package/test/src/broadcast_iterable.test.ts +26 -8
  83. package/test/src/diagnostics.test.ts +128 -0
  84. package/test/src/routes/admin.test.ts +235 -4
  85. package/test/src/routes/sync-rules.test.ts +124 -0
  86. package/test/src/source_table_config.test.ts +104 -0
  87. package/test/src/storage/BucketStorageFactory.test.ts +46 -0
  88. package/test/src/sync/BucketChecksumState.test.ts +195 -6
  89. package/test/src/util/param-logging.test.ts +59 -0
  90. package/tsconfig.tsbuildinfo +1 -1
@@ -16,6 +16,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
16
16
  const { routerEngine, metricsEngine, syncContext } = service_context;
17
17
  const streamStart = Date.now();
18
18
 
19
+ // Format client-controlled values before registering connection resources.
20
+ const formattedAppMetadata = params.app_metadata ? limitParamsForLogging(params.app_metadata) : undefined;
21
+ const formattedClientParams = params.parameters ? limitParamsForLogging(params.parameters) : undefined;
22
+
19
23
  logger.defaultMeta = {
20
24
  ...logger.defaultMeta,
21
25
  user_id: context.token_payload!.userIdJson,
@@ -99,13 +103,12 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
99
103
  tracker.setCompressed(connection.tracker.encoding);
100
104
  }
101
105
 
102
- const formattedAppMetadata = params.app_metadata ? limitParamsForLogging(params.app_metadata) : undefined;
103
- logger.info('Sync stream started', {
104
- app_metadata: formattedAppMetadata,
105
- client_params: params.parameters ? limitParamsForLogging(params.parameters) : undefined
106
- });
107
-
108
106
  try {
107
+ logger.info('Sync stream started', {
108
+ app_metadata: formattedAppMetadata,
109
+ client_params: formattedClientParams
110
+ });
111
+
109
112
  for await (const data of sync.streamResponse({
110
113
  syncContext: syncContext,
111
114
  bucketStorage: bucketStorage,
@@ -2,9 +2,10 @@ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-frame
2
2
  import { SqlSyncRules, SyncConfigWithErrors, SyncRulesErrors } from '@powersync/service-sync-rules';
3
3
  import type { FastifyPluginAsync } from 'fastify';
4
4
  import * as t from 'ts-codec';
5
+ import { assertSourceCapabilities } from '../../api/source-capabilities.js';
5
6
 
6
- import { RouteAPI } from '../../api/RouteAPI.js';
7
- import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
7
+ import { PatternResult, RouteAPI } from '../../api/RouteAPI.js';
8
+ import { logSyncConfigErrors, updateSyncRulesFromConfig } from '../../storage/BucketStorageFactory.js';
8
9
  import { authApi } from '../auth.js';
9
10
  import { routeDefinition } from '../router.js';
10
11
 
@@ -55,6 +56,7 @@ export const deploySyncRules = routeDefinition({
55
56
  let syncConfig: SyncConfigWithErrors;
56
57
 
57
58
  try {
59
+ // First ensure the SyncConfig parses correctly
58
60
  const apiHandler = service_context.routerEngine.getAPI();
59
61
  syncConfig = SqlSyncRules.fromYaml(content, {
60
62
  ...apiHandler.getParseSyncRulesOptions(),
@@ -70,6 +72,10 @@ export const deploySyncRules = routeDefinition({
70
72
  });
71
73
  }
72
74
 
75
+ logSyncConfigErrors(syncConfig);
76
+ // Validate if the current sources support the configuration specified in the sync config
77
+ await assertSourceCapabilities(service_context.routerEngine.getAPI(), syncConfig.config);
78
+
73
79
  const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
74
80
 
75
81
  return {
@@ -176,11 +182,18 @@ export const reprocessSyncRules = routeDefinition({
176
182
  }
177
183
 
178
184
  const sync_rules = active.content;
185
+ const parsed = SqlSyncRules.fromYaml(sync_rules.sync_rules_content, {
186
+ ...payload.context.service_context.routerEngine.getAPI().getParseSyncRulesOptions(),
187
+ schema: undefined,
188
+ // This sync config already passed validation. But if the rules are not valid anymore due
189
+ // to a service change, we do want to report the error here.
190
+ throwOnError: true
191
+ });
192
+ logSyncConfigErrors(parsed);
193
+ await assertSourceCapabilities(payload.context.service_context.routerEngine.getAPI(), parsed.config);
194
+
179
195
  const new_rules = await activeBucketStorage.updateSyncRules(
180
- updateSyncRulesFromYaml(sync_rules.sync_rules_content, {
181
- // This sync config already passed validation. But if the rules are not valid anymore due
182
- // to a service change, we do want to report the error here.
183
- validate: true,
196
+ updateSyncRulesFromConfig(parsed, {
184
197
  version_label: sync_rules.version_label,
185
198
  forceNewReplicationStream: true
186
199
  })
@@ -208,11 +221,31 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
208
221
  // No schema-based validation at this point
209
222
  schema: undefined
210
223
  });
211
- const source_table_patterns = rules.config.getSourceTables();
212
- const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);
224
+ const errors: string[] = [];
225
+ const warnings = rules.errors.filter((error) => error.type === 'warning').map((error) => error.message);
226
+ const recordError = (error: any) => {
227
+ errors.push(...(error instanceof SyncRulesErrors ? error.errors.map((e) => e.message) : [error.message]));
228
+ };
229
+ try {
230
+ const diagnostics = (await apiHandler.validateSourceCapabilities?.(rules.config)) ?? [];
231
+ for (const diagnostic of diagnostics) {
232
+ (diagnostic.level === 'fatal' ? errors : warnings).push(diagnostic.message);
233
+ }
234
+ } catch (e) {
235
+ recordError(e);
236
+ }
237
+ let resolved_tables: PatternResult[] = [];
238
+ try {
239
+ const source_table_patterns = rules.config.getSourceTables();
240
+ resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);
241
+ } catch (e) {
242
+ recordError(e);
243
+ }
213
244
 
214
245
  return {
215
- valid: true,
246
+ valid: errors.length == 0,
247
+ ...(warnings.length > 0 ? { warnings } : {}),
248
+ ...(errors.length > 0 ? { errors } : {}),
216
249
  bucket_definitions: rules.config.debugRepresentation(),
217
250
  source_tables: resolved_tables,
218
251
  data_tables: rules.config.debugGetOutputTables()
@@ -117,10 +117,14 @@ export abstract class BucketStorageFactory
117
117
  */
118
118
  abstract getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
119
119
 
120
- abstract [Symbol.asyncDispose](): PromiseLike<void>;
120
+ async [Symbol.asyncDispose](): Promise<void> {
121
+ await this.iterateAsyncListeners(async (listener) => listener.beforeDispose?.());
122
+ }
121
123
  }
122
124
 
123
125
  export interface BucketStorageFactoryListener {
126
+ /** Release owned resources before the factory closes its database connections. */
127
+ beforeDispose: () => Promise<void>;
124
128
  syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
125
129
  replicationEvent: (event: ReplicationEventPayload) => void;
126
130
  }
@@ -177,7 +181,7 @@ export interface UpdateSyncRulesOptions {
177
181
  config: {
178
182
  yaml: string;
179
183
  /**
180
- * The serialized sync plan for the sync configuration, or `null` for configurations not using the sync stream
184
+ * The serialized sync plan for the sync configuration, or `null` for configurations not using the Sync Streams
181
185
  * compiler.
182
186
  */
183
187
  plan: SerializedSyncPlan | null;
@@ -223,14 +227,15 @@ export function updateSyncRulesFromYaml(
223
227
  content: string,
224
228
  options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
225
229
  ): UpdateSyncRulesOptions {
230
+ const { validate, ...updateOptions } = options ?? {};
226
231
  const config = SqlSyncRules.fromYaml(content, {
227
232
  // No schema-based validation at this point
228
233
  schema: undefined,
229
234
  defaultSchema: options?.defaultSchema ?? 'not_applicable', // Not needed for validation
230
- throwOnError: options?.validate ?? false
235
+ throwOnError: validate ?? false
231
236
  });
232
237
 
233
- return updateSyncRulesFromConfig(config, options);
238
+ return updateSyncRulesFromConfig(config, updateOptions);
234
239
  }
235
240
 
236
241
  export function updateSyncRulesFromConfig(
@@ -257,7 +262,31 @@ export function updateSyncRulesFromConfig(
257
262
  return { config: { yaml: config.content, plan, parsed }, ...options };
258
263
  }
259
264
 
265
+ /**
266
+ * Logs the errors and warnings of a parsed sync config
267
+ *
268
+ * Call this where a sync config is loaded or deployed, so that operators see these in the service logs and not only
269
+ * in the diagnostics and validate API responses.
270
+ *
271
+ * Fatal errors usually throw during parsing before this is reached. They can reach this when the sync config is parsed
272
+ * without validation, such as at startup with `exit_on_error: false`.
273
+ */
274
+ export function logSyncConfigErrors(parsed: SyncConfigWithErrors, log: typeof logger = logger) {
275
+ for (const error of parsed.errors) {
276
+ if (error.type == 'warning') {
277
+ log.warn(`Sync config warning: ${error.message}`);
278
+ } else {
279
+ log.error(`Sync config error: ${error.message}`);
280
+ }
281
+ }
282
+ }
283
+
260
284
  export interface GetIntanceOptions {
285
+ /**
286
+ * The job lease, including a lease acquired during initial configuration.
287
+ * Required for writing; optional for reading.
288
+ */
289
+ replicationLock?: ReplicationLock;
261
290
  /**
262
291
  * Set to true to skip trigger any events for creating the instance.
263
292
  *
@@ -10,6 +10,7 @@ import {
10
10
  HydrationState,
11
11
  nodeSqlite,
12
12
  PrecompiledSyncConfig,
13
+ SourceConnectionMap,
13
14
  SqlSyncRules,
14
15
  SyncConfigWithErrors,
15
16
  versionedHydrationState,
@@ -202,6 +203,8 @@ export interface PersistedSyncConfigContentData {
202
203
  readonly version_label?: string;
203
204
  }
204
205
  export type PersistedSyncConfigId = string;
206
+
205
207
  export interface ParseSyncConfigOptions {
206
208
  defaultSchema: string;
209
+ connections?: SourceConnectionMap;
207
210
  }
@@ -31,5 +31,8 @@ export interface StorageProvider {
31
31
  */
32
32
  type: string;
33
33
 
34
+ /**
35
+ * Creates active storage during storage-engine startup.
36
+ */
34
37
  getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
35
38
  }
@@ -49,11 +49,6 @@ export const STORAGE_VERSION_4 = 4;
49
49
  */
50
50
  export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
51
51
 
52
- /**
53
- * Default storage version for newly persisted replication streams.
54
- */
55
- export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
56
-
57
52
  /**
58
53
  * All versions that can be loaded.
59
54
  *
@@ -11,7 +11,7 @@ import * as util from '../util/util-index.js';
11
11
  import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
12
12
  import { BucketStorageFactory } from './BucketStorageFactory.js';
13
13
  import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
14
- import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
14
+ import type { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
15
15
  import { SourceEntityDescriptor } from './SourceEntity.js';
16
16
  import { SourceTable } from './SourceTable.js';
17
17
  import { SourceTableCandidateReconciler } from './SourceTableReconciler.js';
@@ -402,6 +402,8 @@ export interface CompactOptions {
402
402
  }
403
403
 
404
404
  export interface CompactInitialReplicationOptions {
405
+ /** Internal/testing use: bypass the new-chunk threshold in MongoDB V3/V4 storage. */
406
+ forceChunkCompaction?: boolean;
405
407
  /**
406
408
  * Compact data up to this op id.
407
409
  *
@@ -1,4 +1,4 @@
1
- import { CompatibilityContext } from '@powersync/service-sync-rules';
1
+ import { CompatibilityContext, sourceTableConfigsEqual } from '@powersync/service-sync-rules';
2
2
  import { Logger } from 'winston';
3
3
  import { SerializedSyncPlan, UpdateSyncRulesOptions } from '../BucketStorageFactory.js';
4
4
 
@@ -11,8 +11,8 @@ export function isCompatible(
11
11
  logger: Logger
12
12
  ): boolean {
13
13
  if (updateConfig.plan == null) {
14
- // Only support sync streams with serialized plans
15
- logger.info(`Not using current sync streams - incremental reprocessing not supported`);
14
+ // Only support Sync Streams with serialized plans
15
+ logger.info(`Not using current Sync Streams - incremental reprocessing not supported`);
16
16
  return false;
17
17
  }
18
18
 
@@ -23,8 +23,8 @@ export function isCompatible(
23
23
  }
24
24
 
25
25
  if (existingPlans.some((plan) => plan == null)) {
26
- // Only support sync streams with serialized plans
27
- logger.info(`Existing replication stream not using current sync streams - incremental reprocessing not supported`);
26
+ // Only support Sync Streams with serialized plans
27
+ logger.info(`Existing replication stream not using current Sync Streams - incremental reprocessing not supported`);
28
28
  return false;
29
29
  }
30
30
 
@@ -37,5 +37,17 @@ export function isCompatible(
37
37
  return false;
38
38
  }
39
39
 
40
+ if (
41
+ existingPlans.some(
42
+ (existing) =>
43
+ !sourceTableConfigsEqual(
44
+ existing!.plan.sourceTableConfig ?? {},
45
+ updateConfig.plan!.plan.sourceTableConfig ?? {}
46
+ )
47
+ )
48
+ ) {
49
+ logger.info('Source table configuration changed - incremental reprocessing not supported');
50
+ return false;
51
+ }
40
52
  return true;
41
53
  }
@@ -89,7 +89,7 @@ export class BucketChecksumState {
89
89
  * subscribed to, we keep an index of the stream names to their index in that array.
90
90
  *
91
91
  * This is used to compress the representation of buckets in `checkpoint` and `checkpoint_diff` lines: For buckets
92
- * that are part of sync rules or default streams, we need to include the name of the defining sync rule or definition
92
+ * that are part of legacy Sync Rules or default streams, we need to include the name of the defining bucket definition or stream
93
93
  * yielding that bucket (so that clients can track progress for default streams).
94
94
  * But instead of sending the name for each bucket, we use the fact that it's part of the streams array and only send
95
95
  * their index, reducing the size of those messages.
@@ -747,7 +747,8 @@ export class BucketParameterState {
747
747
  }
748
748
  }
749
749
  }
750
- const allBuckets = [...staticBuckets, ...mergeBuckets(dynamicBuckets)];
750
+ // A stream can reach the same bucket through both static and dynamic parameters.
751
+ const allBuckets = mergeBuckets([...staticBuckets, ...dynamicBuckets]);
751
752
 
752
753
  if (invalidateDataBuckets) {
753
754
  return {
@@ -816,7 +817,7 @@ function logCheckpoint(
816
817
  }
817
818
 
818
819
  /**
819
- * Format a breakdown of dynamic buckets by sync stream or legacy bucket definition.
820
+ * Format a breakdown of dynamic buckets by Sync Stream or legacy bucket definition.
820
821
  *
821
822
  * Sorts definitions by count (descending), includes the top 100, and returns both the
822
823
  * formatted message string and the counts record suitable for structured log data.
@@ -834,7 +835,7 @@ function formatBucketDefinitionBreakdown(
834
835
  const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
835
836
  const sortedDefinitions = allSorted.slice(0, maxLoggedDefinitions);
836
837
 
837
- const sourceLabel = bucketSourceType == BucketSourceType.SYNC_STREAM ? 'sync stream' : 'bucket definition';
838
+ const sourceLabel = bucketSourceType == BucketSourceType.SYNC_STREAM ? 'Sync Stream' : 'bucket definition';
838
839
  let message = `\nBuckets by ${sourceLabel}:`;
839
840
  const countsByDefinition: Record<string, number> = {};
840
841
  for (const [definition, count] of sortedDefinitions) {
@@ -18,8 +18,9 @@ export const DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS: ParamLoggingFormatOptions = {
18
18
  * Formats potentially arbitrary parameters for logging.
19
19
  * This limits the number of keys and strings to a maximum length.
20
20
  * A warning key-value is added if the number of keys exceeds the maximum.
21
- * String values exceeding the maximum length are truncated.
21
+ * Keys and string values exceeding the maximum length are truncated.
22
22
  * Non-String values are stringified, the maximum length is then applied.
23
+ * Values that cannot be serialized are replaced with a placeholder.
23
24
  * @param params - The parameters to format.
24
25
  * @param options - The options to use.
25
26
  * @default DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS
@@ -41,20 +42,26 @@ export function limitParamsForLogging(
41
42
  return value;
42
43
  }
43
44
 
44
- return Object.fromEntries(
45
- Object.entries(params).map(([key, value], index) => {
46
- if (index == maxKeyCount) {
47
- return ['⚠️', 'Additional parameters omitted'];
48
- }
45
+ function formatValue(value: unknown): string {
46
+ if (typeof value == 'string') {
47
+ return trimString(value);
48
+ }
49
+ try {
50
+ return trimString(JSON.stringify(value) ?? '[undefined]');
51
+ } catch {
52
+ return trimString('[unserializable]');
53
+ }
54
+ }
49
55
 
50
- if (index > maxKeyCount) {
51
- return [];
52
- }
56
+ return Object.fromEntries(
57
+ Object.entries(params)
58
+ .slice(0, maxKeyCount + 1)
59
+ .map(([key, value], index) => {
60
+ if (index == maxKeyCount) {
61
+ return ['⚠️', 'Additional parameters omitted'];
62
+ }
53
63
 
54
- if (typeof value == 'string') {
55
- return [key, trimString(value)];
56
- }
57
- return [key, trimString(JSON.stringify(value))];
58
- })
64
+ return [trimString(key), formatValue(value)];
65
+ })
59
66
  );
60
67
  }
@@ -19,7 +19,7 @@ export type BucketRequest = t.Decoded<typeof BucketRequest>;
19
19
  */
20
20
  export const RequestedStreamSubscription = t.object({
21
21
  /**
22
- * The defined name of the stream as it appears in sync stream definitions.
22
+ * The defined name of the stream as it appears in Sync Stream definitions.
23
23
  */
24
24
  stream: t.string,
25
25
  /**
@@ -29,7 +29,7 @@ export const RequestedStreamSubscription = t.object({
29
29
  /**
30
30
  * Set when the client wishes to re-assign a different priority to this stream.
31
31
  *
32
- * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
32
+ * Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
33
33
  * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
34
34
  */
35
35
  override_priority: orNull(t.number)
@@ -49,7 +49,7 @@ export const StreamSubscriptionRequest = t.object({
49
49
  include_defaults: t.boolean.optional(),
50
50
 
51
51
  /**
52
- * An array of sync streams the client has opened explicitly.
52
+ * An array of Sync Streams the client has opened explicitly.
53
53
  */
54
54
  subscriptions: t.array(RequestedStreamSubscription)
55
55
  });
@@ -253,8 +253,8 @@ export interface BucketChecksum {
253
253
  /**
254
254
  * The reason a particular bucket is included in a checkpoint.
255
255
  *
256
- * This information allows clients to associate individual buckets with sync streams they're subscribed to. Having that
257
- * association is useful because it enables clients to track progress for individual sync streams.
256
+ * This information allows clients to associate individual buckets with Sync Streams they're subscribed to. Having that
257
+ * association is useful because it enables clients to track progress for individual Sync Streams.
258
258
  */
259
259
  export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
260
260
 
@@ -1,6 +1,9 @@
1
+ import type { RouteAPI } from '@/api/RouteAPI.js';
2
+ import { assertSourceCapabilities } from '@/api/source-capabilities.js';
1
3
  import { AbstractReplicationJob } from '@/replication/AbstractReplicationJob.js';
2
4
  import { AbstractReplicator, AbstractReplicatorOptions, CreateJobOptions } from '@/replication/AbstractReplicator.js';
3
5
  import { PersistedReplicationStream } from '@/storage/PersistedReplicationStream.js';
6
+ import type { ReplicationLock } from '@/storage/ReplicationLock.js';
4
7
  import { SyncRulesBucketStorage } from '@/storage/SyncRulesBucketStorage.js';
5
8
  import { describe, expect, it, vi } from 'vitest';
6
9
 
@@ -45,6 +48,21 @@ class TestReplicator extends AbstractReplicator {
45
48
  return (this as any).heartbeatIntervalNanos;
46
49
  }
47
50
 
51
+ async runStartupForTest(): Promise<void> {
52
+ const controller = new AbortController();
53
+ controller.abort();
54
+ (this as any).abortController = controller;
55
+ await (this as any).runLoop();
56
+ }
57
+
58
+ async refreshForTest(configuredLock?: ReplicationLock): Promise<{ replicationJobStarted: boolean }> {
59
+ (this as any).abortController = new AbortController();
60
+ return (this as any).refresh({
61
+ configuredLock,
62
+ loadedVersionLabel: this.syncRuleProvider.versionLabel
63
+ });
64
+ }
65
+
48
66
  shouldHandleStreamForTest(
49
67
  replicationStream: PersistedReplicationStream,
50
68
  loadedSyncRules: string | undefined,
@@ -54,6 +72,144 @@ class TestReplicator extends AbstractReplicator {
54
72
  }
55
73
  }
56
74
 
75
+ describe('AbstractReplicator startup sync config', () => {
76
+ it.each([true, false])('respects exit_on_error=%s for unsupported file-loaded filters', async (exitOnError) => {
77
+ const yaml = /* yaml */ `
78
+ # sync config fixture.
79
+ config:
80
+ edition: 3
81
+ source_table_options:
82
+ orders:
83
+ mongodb_filter_expression: { $eq: ['$$doc.active', true] }
84
+ streams:
85
+ orders:
86
+ query: SELECT * FROM orders
87
+ `;
88
+ const configureSyncRules = vi.fn(
89
+ async (_options: import('@/storage/BucketStorageFactory.js').UpdateSyncRulesOptions) => ({ updated: false })
90
+ );
91
+ const replicator = new TestReplicator(async () => {}, {
92
+ id: 'test',
93
+ storageEngine: {
94
+ activeBucketStorage: { configureSyncRules }
95
+ } as unknown as AbstractReplicatorOptions['storageEngine'],
96
+ configuration: { connections: [{ type: 'postgresql' }] },
97
+ syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: undefined },
98
+ metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
99
+ rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
100
+ });
101
+ if (exitOnError) {
102
+ await expect(replicator.runStartupForTest()).rejects.toThrow('only supported by MongoDB sources');
103
+ expect(configureSyncRules).not.toHaveBeenCalled();
104
+ } else {
105
+ await expect(replicator.runStartupForTest()).resolves.toBeUndefined();
106
+ expect(configureSyncRules).toHaveBeenCalledOnce();
107
+ expect(configureSyncRules.mock.calls[0][0].config.plan?.errors).toEqual([
108
+ expect.objectContaining({
109
+ level: 'fatal',
110
+ message: expect.stringContaining('only supported by MongoDB sources')
111
+ })
112
+ ]);
113
+ }
114
+ });
115
+
116
+ it.each(
117
+ [true, false].flatMap((exitOnError) =>
118
+ (['warning', 'valid', 'unexpected error'] as const).map((result) => ({ exitOnError, result }))
119
+ )
120
+ )(
121
+ 'checks source capabilities before persistence: $result, exit_on_error=$exitOnError',
122
+ async ({ result, exitOnError }) => {
123
+ const yaml = /* yaml */ `
124
+ # sync config fixture.
125
+ config:
126
+ edition: 3
127
+ streams:
128
+ orders:
129
+ query: SELECT * FROM orders
130
+ `;
131
+ const calls: string[] = [];
132
+ const configureSyncRules = vi.fn(async () => {
133
+ calls.push('persist');
134
+ return { updated: false };
135
+ });
136
+ const replicator = new TestReplicator(async () => {}, {
137
+ id: 'test',
138
+ storageEngine: {
139
+ activeBucketStorage: { configureSyncRules }
140
+ } as unknown as AbstractReplicatorOptions['storageEngine'],
141
+ syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: undefined },
142
+ metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
143
+ rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
144
+ });
145
+ replicator.registerSourceCapabilitiesAssertion((config) =>
146
+ assertSourceCapabilities(
147
+ {
148
+ async validateSourceCapabilities() {
149
+ calls.push('validate');
150
+ if (result === 'unexpected error') throw new Error('Source validation failed');
151
+ return result === 'warning' ? [{ level: 'warning', message: 'Missing index' }] : [];
152
+ }
153
+ } as unknown as RouteAPI,
154
+ config
155
+ )
156
+ );
157
+
158
+ if (result === 'unexpected error' && exitOnError) {
159
+ await expect(replicator.runStartupForTest()).rejects.toThrow('Source validation failed');
160
+ expect(calls).toEqual(['validate']);
161
+ } else {
162
+ await replicator.runStartupForTest();
163
+ expect(calls).toEqual(['validate', 'persist']);
164
+ }
165
+ }
166
+ );
167
+
168
+ it.each([true, false])('retains source-table options with exit_on_error=%s', async (exitOnError) => {
169
+ const yaml = /* yaml */ `
170
+ {
171
+ config: { edition: 3, source_table_options: { orders: { mongodb_filter_expression: 'disabled' } } },
172
+ streams: { orders: { query: SELECT * FROM orders } }
173
+ }
174
+ `;
175
+ const configureSyncRules = vi.fn(async () => ({ updated: false }));
176
+ const replicator = new TestReplicator(async () => {}, {
177
+ id: 'test',
178
+ storageEngine: {
179
+ activeBucketStorage: { configureSyncRules }
180
+ } as unknown as AbstractReplicatorOptions['storageEngine'],
181
+ syncRuleProvider: { get: async () => yaml, exitOnError, versionLabel: 'v2' },
182
+ metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
183
+ rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
184
+ });
185
+
186
+ await replicator.runStartupForTest();
187
+
188
+ expect(configureSyncRules).toHaveBeenCalledExactlyOnceWith(
189
+ expect.objectContaining({
190
+ lock: true,
191
+ version_label: 'v2',
192
+ config: expect.objectContaining({
193
+ parsed: expect.objectContaining({
194
+ errors: [],
195
+ config: expect.objectContaining({
196
+ plan: expect.objectContaining({
197
+ sourceTableConfig: { orders: { mongodb_filter_expression: 'disabled' } }
198
+ })
199
+ })
200
+ }),
201
+ plan: expect.objectContaining({
202
+ plan: expect.objectContaining({
203
+ version: 3,
204
+ sourceTableConfig: { orders: { mongodb_filter_expression: 'disabled' } }
205
+ })
206
+ })
207
+ })
208
+ })
209
+ );
210
+ });
211
+ });
212
+
57
213
  describe('AbstractReplicator heartbeat interval', () => {
58
214
  const options: AbstractReplicatorOptions = {
59
215
  id: 'test',