@powersync/service-core 1.26.1 → 1.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +67 -0
  2. package/dist/api/RouteAPI.d.ts +8 -1
  3. package/dist/api/api-index.d.ts +1 -0
  4. package/dist/api/api-index.js +1 -0
  5. package/dist/api/api-index.js.map +1 -1
  6. package/dist/api/diagnostics.js +33 -16
  7. package/dist/api/diagnostics.js.map +1 -1
  8. package/dist/api/source-capabilities.d.ts +7 -0
  9. package/dist/api/source-capabilities.js +23 -0
  10. package/dist/api/source-capabilities.js.map +1 -0
  11. package/dist/auth/JwtPayload.d.ts +2 -2
  12. package/dist/auth/JwtPayload.js +2 -2
  13. package/dist/auth/JwtPayload.js.map +1 -1
  14. package/dist/entry/commands/compact-action.js +1 -1
  15. package/dist/entry/commands/compact-action.js.map +1 -1
  16. package/dist/entry/commands/config-command.js +1 -1
  17. package/dist/entry/commands/config-command.js.map +1 -1
  18. package/dist/migrations/PowerSyncMigrationManager.js +2 -1
  19. package/dist/migrations/PowerSyncMigrationManager.js.map +1 -1
  20. package/dist/replication/AbstractReplicationJob.d.ts +4 -6
  21. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  22. package/dist/replication/AbstractReplicator.d.ts +15 -0
  23. package/dist/replication/AbstractReplicator.js +37 -5
  24. package/dist/replication/AbstractReplicator.js.map +1 -1
  25. package/dist/replication/ReplicationModule.d.ts +0 -7
  26. package/dist/replication/ReplicationModule.js +13 -3
  27. package/dist/replication/ReplicationModule.js.map +1 -1
  28. package/dist/routes/configure-fastify.d.ts +39 -27
  29. package/dist/routes/endpoints/admin.d.ts +78 -54
  30. package/dist/routes/endpoints/admin.js +24 -10
  31. package/dist/routes/endpoints/admin.js.map +1 -1
  32. package/dist/routes/endpoints/socket-route.js +7 -5
  33. package/dist/routes/endpoints/socket-route.js.map +1 -1
  34. package/dist/routes/endpoints/sync-rules.js +39 -6
  35. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  36. package/dist/storage/BucketStorageFactory.d.ts +20 -3
  37. package/dist/storage/BucketStorageFactory.js +25 -2
  38. package/dist/storage/BucketStorageFactory.js.map +1 -1
  39. package/dist/storage/PersistedSyncConfigContent.d.ts +2 -1
  40. package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
  41. package/dist/storage/StorageProvider.d.ts +3 -0
  42. package/dist/storage/StorageVersionConfig.d.ts +0 -4
  43. package/dist/storage/StorageVersionConfig.js +0 -4
  44. package/dist/storage/StorageVersionConfig.js.map +1 -1
  45. package/dist/storage/SyncRulesBucketStorage.d.ts +3 -1
  46. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  47. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +9 -5
  48. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -1
  49. package/dist/sync/BucketChecksumState.d.ts +1 -1
  50. package/dist/sync/BucketChecksumState.js +5 -4
  51. package/dist/sync/BucketChecksumState.js.map +1 -1
  52. package/dist/util/param-logging.d.ts +5 -2
  53. package/dist/util/param-logging.js +17 -9
  54. package/dist/util/param-logging.js.map +1 -1
  55. package/dist/util/protocol-types.d.ts +7 -7
  56. package/dist/util/protocol-types.js +3 -3
  57. package/package.json +11 -11
  58. package/src/api/RouteAPI.ts +9 -1
  59. package/src/api/api-index.ts +1 -0
  60. package/src/api/diagnostics.ts +30 -14
  61. package/src/api/source-capabilities.ts +24 -0
  62. package/src/auth/JwtPayload.ts +2 -2
  63. package/src/entry/commands/compact-action.ts +1 -1
  64. package/src/entry/commands/config-command.ts +1 -1
  65. package/src/migrations/PowerSyncMigrationManager.ts +2 -1
  66. package/src/replication/AbstractReplicationJob.ts +2 -4
  67. package/src/replication/AbstractReplicator.ts +46 -5
  68. package/src/replication/ReplicationModule.ts +12 -12
  69. package/src/routes/endpoints/admin.ts +29 -11
  70. package/src/routes/endpoints/socket-route.ts +9 -6
  71. package/src/routes/endpoints/sync-rules.ts +42 -9
  72. package/src/storage/BucketStorageFactory.ts +33 -4
  73. package/src/storage/PersistedSyncConfigContent.ts +3 -0
  74. package/src/storage/StorageProvider.ts +3 -0
  75. package/src/storage/StorageVersionConfig.ts +0 -5
  76. package/src/storage/SyncRulesBucketStorage.ts +3 -1
  77. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +17 -5
  78. package/src/sync/BucketChecksumState.ts +5 -4
  79. package/src/util/param-logging.ts +21 -14
  80. package/src/util/protocol-types.ts +5 -5
  81. package/test/src/AbstractReplicator.test.ts +156 -0
  82. package/test/src/broadcast_iterable.test.ts +26 -8
  83. package/test/src/diagnostics.test.ts +128 -0
  84. package/test/src/routes/admin.test.ts +235 -4
  85. package/test/src/routes/sync-rules.test.ts +124 -0
  86. package/test/src/source_table_config.test.ts +104 -0
  87. package/test/src/storage/BucketStorageFactory.test.ts +46 -0
  88. package/test/src/sync/BucketChecksumState.test.ts +195 -6
  89. package/test/src/util/param-logging.test.ts +59 -0
  90. package/tsconfig.tsbuildinfo +1 -1
@@ -9,8 +9,9 @@ export const DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS = {
9
9
  * Formats potentially arbitrary parameters for logging.
10
10
  * This limits the number of keys and strings to a maximum length.
11
11
  * A warning key-value is added if the number of keys exceeds the maximum.
12
- * String values exceeding the maximum length are truncated.
12
+ * Keys and string values exceeding the maximum length are truncated.
13
13
  * Non-String values are stringified, the maximum length is then applied.
14
+ * Values that cannot be serialized are replaced with a placeholder.
14
15
  * @param params - The parameters to format.
15
16
  * @param options - The options to use.
16
17
  * @default DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS
@@ -24,17 +25,24 @@ export function limitParamsForLogging(params, options = DEFAULT_PARAM_LOGGING_FO
24
25
  }
25
26
  return value;
26
27
  }
27
- return Object.fromEntries(Object.entries(params).map(([key, value], index) => {
28
- if (index == maxKeyCount) {
29
- return ['⚠️', 'Additional parameters omitted'];
28
+ function formatValue(value) {
29
+ if (typeof value == 'string') {
30
+ return trimString(value);
30
31
  }
31
- if (index > maxKeyCount) {
32
- return [];
32
+ try {
33
+ return trimString(JSON.stringify(value) ?? '[undefined]');
33
34
  }
34
- if (typeof value == 'string') {
35
- return [key, trimString(value)];
35
+ catch {
36
+ return trimString('[unserializable]');
37
+ }
38
+ }
39
+ return Object.fromEntries(Object.entries(params)
40
+ .slice(0, maxKeyCount + 1)
41
+ .map(([key, value], index) => {
42
+ if (index == maxKeyCount) {
43
+ return ['⚠️', 'Additional parameters omitted'];
36
44
  }
37
- return [key, trimString(JSON.stringify(value))];
45
+ return [trimString(key), formatValue(value)];
38
46
  }));
39
47
  }
40
48
  //# sourceMappingURL=param-logging.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"param-logging.js","sourceRoot":"","sources":["../../src/util/param-logging.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAA8B;IAC7E,WAAW,EAAE,EAAE;IACf,eAAe,EAAE,GAAG;CACrB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAA2B,EAC3B,UAA8C,oCAAoC;IAElF,MAAM,EACJ,eAAe,GAAG,oCAAoC,CAAC,eAAe,EACtE,WAAW,GAAG,oCAAoC,CAAC,WAAW,EAC/D,GAAG,OAAO,CAAC;IAEZ,SAAS,UAAU,CAAC,KAAa;QAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;QACjD,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,EAAE,+BAA+B,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"param-logging.js","sourceRoot":"","sources":["../../src/util/param-logging.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAA8B;IAC7E,WAAW,EAAE,EAAE;IACf,eAAe,EAAE,GAAG;CACrB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAA2B,EAC3B,UAA8C,oCAAoC;IAElF,MAAM,EACJ,eAAe,GAAG,oCAAoC,CAAC,eAAe,EACtE,WAAW,GAAG,oCAAoC,CAAC,WAAW,EAC/D,GAAG,OAAO,CAAC;IAEZ,SAAS,UAAU,CAAC,KAAa;QAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,WAAW,CAAC,KAAc;QACjC,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC;YACH,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACnB,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;QAC3B,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,EAAE,+BAA+B,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CACL,CAAC;AACJ,CAAC"}
@@ -14,7 +14,7 @@ export type BucketRequest = t.Decoded<typeof BucketRequest>;
14
14
  */
15
15
  export declare const RequestedStreamSubscription: t.ObjectCodec<{
16
16
  /**
17
- * The defined name of the stream as it appears in sync stream definitions.
17
+ * The defined name of the stream as it appears in Sync Stream definitions.
18
18
  */
19
19
  stream: t.IdentityCodec<t.CodecType.String>;
20
20
  /**
@@ -24,7 +24,7 @@ export declare const RequestedStreamSubscription: t.ObjectCodec<{
24
24
  /**
25
25
  * Set when the client wishes to re-assign a different priority to this stream.
26
26
  *
27
- * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
27
+ * Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
28
28
  * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
29
29
  */
30
30
  override_priority: t.Codec<number | null, number | null>;
@@ -41,11 +41,11 @@ export declare const StreamSubscriptionRequest: t.ObjectCodec<{
41
41
  */
42
42
  include_defaults: t.OptionalCodec<t.Codec<boolean, boolean, string, t.CodecProps>>;
43
43
  /**
44
- * An array of sync streams the client has opened explicitly.
44
+ * An array of Sync Streams the client has opened explicitly.
45
45
  */
46
46
  subscriptions: t.ArrayCodec<t.ObjectCodec<{
47
47
  /**
48
- * The defined name of the stream as it appears in sync stream definitions.
48
+ * The defined name of the stream as it appears in Sync Stream definitions.
49
49
  */
50
50
  stream: t.IdentityCodec<t.CodecType.String>;
51
51
  /**
@@ -55,7 +55,7 @@ export declare const StreamSubscriptionRequest: t.ObjectCodec<{
55
55
  /**
56
56
  * Set when the client wishes to re-assign a different priority to this stream.
57
57
  *
58
- * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
58
+ * Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
59
59
  * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
60
60
  */
61
61
  override_priority: t.Codec<number | null, number | null>;
@@ -239,8 +239,8 @@ export interface BucketChecksum {
239
239
  /**
240
240
  * The reason a particular bucket is included in a checkpoint.
241
241
  *
242
- * This information allows clients to associate individual buckets with sync streams they're subscribed to. Having that
243
- * association is useful because it enables clients to track progress for individual sync streams.
242
+ * This information allows clients to associate individual buckets with Sync Streams they're subscribed to. Having that
243
+ * association is useful because it enables clients to track progress for individual Sync Streams.
244
244
  */
245
245
  export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
246
246
  /**
@@ -12,7 +12,7 @@ export const BucketRequest = t.object({
12
12
  */
13
13
  export const RequestedStreamSubscription = t.object({
14
14
  /**
15
- * The defined name of the stream as it appears in sync stream definitions.
15
+ * The defined name of the stream as it appears in Sync Stream definitions.
16
16
  */
17
17
  stream: t.string,
18
18
  /**
@@ -22,7 +22,7 @@ export const RequestedStreamSubscription = t.object({
22
22
  /**
23
23
  * Set when the client wishes to re-assign a different priority to this stream.
24
24
  *
25
- * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
25
+ * Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
26
26
  * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
27
27
  */
28
28
  override_priority: orNull(t.number)
@@ -38,7 +38,7 @@ export const StreamSubscriptionRequest = t.object({
38
38
  */
39
39
  include_defaults: t.boolean.optional(),
40
40
  /**
41
- * An array of sync streams the client has opened explicitly.
41
+ * An array of Sync Streams the client has opened explicitly.
42
42
  */
43
43
  subscriptions: t.array(RequestedStreamSubscription)
44
44
  });
package/package.json CHANGED
@@ -4,18 +4,18 @@
4
4
  "publishConfig": {
5
5
  "access": "public"
6
6
  },
7
- "version": "1.26.1",
7
+ "version": "1.27.1",
8
8
  "license": "FSL-1.1-ALv2",
9
9
  "type": "module",
10
10
  "exports": "./dist/index.js",
11
11
  "dependencies": {
12
12
  "@js-sdsl/ordered-set": "^4.4.2",
13
13
  "@opentelemetry/api": "^1.9.0",
14
- "@opentelemetry/core": "^2.0.1",
15
- "@opentelemetry/exporter-metrics-otlp-http": "^0.203.0",
16
- "@opentelemetry/exporter-prometheus": "^0.203.0",
17
- "@opentelemetry/resources": "^2.0.1",
18
- "@opentelemetry/sdk-metrics": "^2.0.1",
14
+ "@opentelemetry/core": "^2.8.0",
15
+ "@opentelemetry/exporter-metrics-otlp-http": "^0.219.0",
16
+ "@opentelemetry/exporter-prometheus": "^0.219.0",
17
+ "@opentelemetry/resources": "^2.8.0",
18
+ "@opentelemetry/sdk-metrics": "^2.8.0",
19
19
  "async": "^3.2.4",
20
20
  "async-mutex": "^0.5.0",
21
21
  "bson": "^6.10.4",
@@ -33,17 +33,17 @@
33
33
  "uuid": "^14.0.0",
34
34
  "winston": "^3.13.0",
35
35
  "yaml": "^2.8.3",
36
- "@powersync/lib-services-framework": "0.10.2",
36
+ "@powersync/service-rsocket-router": "0.2.29",
37
37
  "@powersync/service-jsonbig": "0.17.13",
38
- "@powersync/service-rsocket-router": "0.2.27",
39
- "@powersync/service-types": "0.18.0",
40
- "@powersync/service-sync-rules": "0.42.0"
38
+ "@powersync/service-types": "0.20.0",
39
+ "@powersync/service-sync-rules": "0.44.0",
40
+ "@powersync/lib-services-framework": "0.10.4"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/async": "^3.2.24",
44
44
  "@types/lodash": "^4.17.5",
45
45
  "@types/negotiator": "^0.6.4",
46
- "fastify": "^5.8.5",
46
+ "fastify": "^5.12.5",
47
47
  "fastify-plugin": "^5.0.1"
48
48
  },
49
49
  "scripts": {
@@ -1,6 +1,7 @@
1
1
  import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
2
2
  import * as types from '@powersync/service-types';
3
- import { ParseSyncConfigOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
3
+ import type { ParseSyncConfigOptions } from '../storage/PersistedSyncConfigContent.js';
4
+ import { SyncRulesBucketStorage } from '../storage/storage-index.js';
4
5
 
5
6
  export interface PatternResult {
6
7
  schema: string;
@@ -40,6 +41,13 @@ export interface RouteAPI {
40
41
  */
41
42
  getConnectionStatus(): Promise<types.ConnectionStatusV2>;
42
43
 
44
+ /**
45
+ * Checks whether the source supports the capabilities required by this sync config.
46
+ * Must not modify the source. Return advisory warnings and fatal validation diagnostics.
47
+ * An empty array means no issues; throw only for unexpected failures.
48
+ */
49
+ validateSourceCapabilities?(syncConfig: SyncConfig): Promise<types.ValidationDiagnostic[]>;
50
+
43
51
  /**
44
52
  * Expand sync config table patterns into table metadata from the source connection.
45
53
  *
@@ -2,3 +2,4 @@ export * from './api-metrics.js';
2
2
  export * from './diagnostics.js';
3
3
  export * from './RouteAPI.js';
4
4
  export * from './schema.js';
5
+ export * from './source-capabilities.js';
@@ -1,5 +1,5 @@
1
1
  import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
2
- import { DEFAULT_TAG, SourceTableRef, SyncConfigWithErrors } from '@powersync/service-sync-rules';
2
+ import { DEFAULT_TAG, SourceTableRef, SyncConfigWithErrors, SyncRulesErrors } from '@powersync/service-sync-rules';
3
3
  import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types';
4
4
 
5
5
  import * as storage from '../storage/storage-index.js';
@@ -68,7 +68,11 @@ export async function getSyncRulesStatus(
68
68
  version_label: syncConfig.version_label,
69
69
  content: include_content ? syncConfig.sync_rules_content : undefined,
70
70
  connections: [],
71
- errors: [{ level: 'fatal', message: e.message, ts: now }]
71
+ // Report each sync config error separately, retaining its source location.
72
+ errors:
73
+ e instanceof SyncRulesErrors
74
+ ? e.errors.map((error) => syncConfigYamlErrorToReplicationError(error, now))
75
+ : [{ level: 'fatal', message: e.message, ts: now }]
72
76
  };
73
77
  }
74
78
 
@@ -81,20 +85,32 @@ export async function getSyncRulesStatus(
81
85
  let replication_lag_bytes: number | undefined = undefined;
82
86
  let slot_wal_budget: SlotWalBudgetInfo | undefined = undefined;
83
87
 
88
+ const validationErrors: ReplicationError[] = [];
84
89
  let tables_flat: TableInfo[] = [];
85
90
 
86
91
  if (check_connection) {
87
- const source_table_patterns = rules.getSourceTables();
88
- const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
89
- tables_flat = resolved_tables.flatMap((info) => {
90
- if (info.table) {
91
- return [info.table];
92
- } else if (info.tables) {
93
- return info.tables;
94
- } else {
95
- return [];
96
- }
97
- });
92
+ try {
93
+ validationErrors.push(...((await apiHandler.validateSourceCapabilities?.(rules)) ?? []));
94
+ } catch (e) {
95
+ validationErrors.push({ level: 'fatal', message: e.message, ts: now });
96
+ }
97
+ // Table inspection is independent of source capability checks. Preserve both failures
98
+ // and continue collecting diagnostics that do not depend on table metadata.
99
+ try {
100
+ const source_table_patterns = rules.getSourceTables();
101
+ const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
102
+ tables_flat = resolved_tables.flatMap((info) => {
103
+ if (info.table) {
104
+ return [info.table];
105
+ } else if (info.tables) {
106
+ return info.tables;
107
+ } else {
108
+ return [];
109
+ }
110
+ });
111
+ } catch (e) {
112
+ validationErrors.push({ level: 'fatal', message: e.message, ts: now });
113
+ }
98
114
 
99
115
  if (systemStorage) {
100
116
  try {
@@ -151,7 +167,7 @@ export async function getSyncRulesStatus(
151
167
  });
152
168
  }
153
169
 
154
- const errors = tables_flat.flatMap((info) => info.errors);
170
+ const errors = [...validationErrors, ...tables_flat.flatMap((info) => info.errors)];
155
171
  const statusSource = await syncConfig.getSyncConfigStatus();
156
172
 
157
173
  if (statusSource?.last_fatal_error) {
@@ -0,0 +1,24 @@
1
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
+ import type { SyncConfig } from '@powersync/service-sync-rules';
3
+ import type { RouteAPI } from './RouteAPI.js';
4
+
5
+ /**
6
+ * Blocks deployment and reprocessing on fatal source diagnostics; warnings remain advisory.
7
+ * Capability failures use the same typed 422 error for deployment and reprocessing.
8
+ */
9
+ export async function assertSourceCapabilities(api: RouteAPI, config: SyncConfig): Promise<void> {
10
+ try {
11
+ const diagnostics = (await api.validateSourceCapabilities?.(config)) ?? [];
12
+ const errors = diagnostics.filter((diagnostic) => diagnostic.level === 'fatal');
13
+ if (errors.length > 0) {
14
+ throw new Error(errors.map((error) => error.message).join('\n'));
15
+ }
16
+ } catch (error) {
17
+ throw new ServiceError({
18
+ status: 422,
19
+ code: ErrorCode.PSYNC_R0001,
20
+ description: 'Source capability validation failed',
21
+ details: error.message
22
+ });
23
+ }
24
+ }
@@ -1,9 +1,9 @@
1
- import { BaseJwtPayload } from '@powersync/service-sync-rules';
1
+ import { RequestJwtPayload } from '@powersync/service-sync-rules';
2
2
 
3
3
  /**
4
4
  * Payload from a JWT, always signed.
5
5
  */
6
- export class JwtPayload extends BaseJwtPayload {
6
+ export class JwtPayload extends RequestJwtPayload {
7
7
  /**
8
8
  * Stringified version of sub. Used where we need a string user identifier, such as in write checkpoints
9
9
  * and per-user metrics.
@@ -46,7 +46,7 @@ export function registerCompactAction(program: Command) {
46
46
  if (invalid.length > 0) {
47
47
  logger.error(
48
48
  `Invalid bucket names: ${invalid.join(', ')}. ` +
49
- `Pass full bucket names (e.g., "global[]"), not bucket definition names (e.g., "global").`
49
+ `Pass full bucket names (e.g., "global[]"), not bucket definition or Sync Stream names (e.g., "global").`
50
50
  );
51
51
  process.exit(1);
52
52
  }
@@ -19,7 +19,7 @@ export function wrapConfigCommand(command: Command) {
19
19
  )
20
20
  .option(
21
21
  `-sync64, --sync-base64 [base64]`,
22
- 'Base64 encoded YAML Sync Config. Defaults to process.env.POWERSYNC_SYNC_CONFIG_B64 or process.env.POWERSYNC_SYNC_RULES_B64 for backwards compatility.',
22
+ 'Base64 encoded YAML Sync Config. Defaults to process.env.POWERSYNC_SYNC_CONFIG_B64 or process.env.POWERSYNC_SYNC_RULES_B64 for backwards compatibility.',
23
23
  util.env.POWERSYNC_SYNC_CONFIG_B64 || util.env.POWERSYNC_SYNC_RULES_B64
24
24
  );
25
25
  }
@@ -1,6 +1,7 @@
1
1
  import * as framework from '@powersync/lib-services-framework';
2
2
  import fs from 'fs/promises';
3
3
  import path from 'path';
4
+ import { pathToFileURL } from 'url';
4
5
  import * as system from '../system/system-index.js';
5
6
 
6
7
  /**
@@ -28,7 +29,7 @@ export abstract class AbstractPowerSyncMigrationAgent extends framework.Abstract
28
29
 
29
30
  return await Promise.all(
30
31
  migrations.map(async (migration) => {
31
- const module = await import(path.resolve(migrationsDir, migration));
32
+ const module = await import(pathToFileURL(path.resolve(migrationsDir, migration)).href);
32
33
  return {
33
34
  name: path.basename(migration).replace(path.extname(migration), ''),
34
35
  up: module.up,
@@ -1,14 +1,12 @@
1
1
  import { container, logger } from '@powersync/lib-services-framework';
2
2
  import winston from 'winston';
3
3
  import { MetricsEngine } from '../metrics/MetricsEngine.js';
4
- import * as storage from '../storage/storage-index.js';
4
+ import type { CreateJobOptions } from './AbstractReplicator.js';
5
5
  import { ErrorRateLimiter } from './ErrorRateLimiter.js';
6
6
 
7
- export interface AbstractReplicationJobOptions {
7
+ export interface AbstractReplicationJobOptions extends CreateJobOptions {
8
8
  id: string;
9
- storage: storage.SyncRulesBucketStorage;
10
9
  metrics: MetricsEngine;
11
- lock: storage.ReplicationLock;
12
10
  rateLimiter: ErrorRateLimiter;
13
11
  }
14
12
 
@@ -1,4 +1,5 @@
1
1
  import { container, ErrorCode, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
2
+ import { DEFAULT_TAG, SqlSyncRules, type SyncConfig } from '@powersync/service-sync-rules';
2
3
  import { ReplicationMetric } from '@powersync/service-types';
3
4
  import { hrtime } from 'node:process';
4
5
  import { setTimeout as sleep } from 'node:timers/promises';
@@ -7,6 +8,7 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js';
7
8
  import * as storage from '../storage/storage-index.js';
8
9
  import { StorageEngine } from '../storage/storage-index.js';
9
10
  import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
11
+ import type { ResolvedPowerSyncConfig } from '../util/config/types.js';
10
12
  import { AbstractReplicationJob } from './AbstractReplicationJob.js';
11
13
  import { ErrorRateLimiter } from './ErrorRateLimiter.js';
12
14
  import { ConnectionTestResult } from './ReplicationModule.js';
@@ -30,6 +32,10 @@ export interface AbstractReplicatorOptions {
30
32
  storageEngine: StorageEngine;
31
33
  metricsEngine: MetricsEngine;
32
34
  syncRuleProvider: SyncRulesProvider;
35
+ /**
36
+ * Source connection metadata from the service config, used when parsing filesystem-loaded sync configs.
37
+ */
38
+ configuration?: Pick<ResolvedPowerSyncConfig, 'connections'>;
33
39
  /**
34
40
  * This limits the effect of retries when there is a persistent issue.
35
41
  */
@@ -73,6 +79,8 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
73
79
 
74
80
  private abortController: AbortController | undefined;
75
81
 
82
+ private assertSourceCapabilities: ((config: SyncConfig) => Promise<void>) | undefined;
83
+
76
84
  protected constructor(private options: AbstractReplicatorOptions) {
77
85
  this.logger = logger.child({ name: `Replicator:${options.id}` });
78
86
  const heartbeatIntervalSeconds = options.heartbeatIntervalSeconds ?? DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
@@ -105,6 +113,13 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
105
113
  return this.options.syncRuleProvider;
106
114
  }
107
115
 
116
+ /**
117
+ * Default schema for startup sync config validation. Source modules with a known default should override this.
118
+ */
119
+ protected get defaultSchema(): string {
120
+ return 'not_applicable';
121
+ }
122
+
108
123
  protected get rateLimiter() {
109
124
  return this.options.rateLimiter;
110
125
  }
@@ -117,6 +132,13 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
117
132
  return this.abortController?.signal.aborted;
118
133
  }
119
134
 
135
+ /**
136
+ * Register an assertion that blocks unsupported file-loaded sync configs before replication.
137
+ */
138
+ public registerSourceCapabilitiesAssertion(assertion: (config: SyncConfig) => Promise<void>): void {
139
+ this.assertSourceCapabilities = assertion;
140
+ }
141
+
120
142
  public async start(): Promise<void> {
121
143
  this.abortController = new AbortController();
122
144
  this.runLoop().catch((e) => {
@@ -165,10 +187,30 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
165
187
  // versions before that check was added, so we keep the lock for now - for where te service version and sync config is updated at
166
188
  // the same time.
167
189
 
190
+ const parsed = SqlSyncRules.fromYaml(loadedSyncConfig, {
191
+ schema: undefined,
192
+ defaultSchema: this.defaultSchema,
193
+ connections:
194
+ this.options.configuration?.connections &&
195
+ Object.fromEntries(
196
+ this.options.configuration.connections.map((connection) => [
197
+ connection.tag ?? DEFAULT_TAG,
198
+ { type: connection.type }
199
+ ])
200
+ ),
201
+ throwOnError: this.syncRuleProvider.exitOnError
202
+ });
203
+ storage.logSyncConfigErrors(parsed, this.logger);
204
+ try {
205
+ await this.assertSourceCapabilities?.(parsed.config);
206
+ } catch (error) {
207
+ if (this.syncRuleProvider.exitOnError) throw error;
208
+ // Persist the config for diagnostics; the job-start check will prevent unsafe replication.
209
+ this.logger.error('Source capability validation failed for the sync config', error);
210
+ }
168
211
  const { lock } = await this.storage.configureSyncRules(
169
- storage.updateSyncRulesFromYaml(loadedSyncConfig, {
212
+ storage.updateSyncRulesFromConfig(parsed, {
170
213
  lock: true,
171
- validate: this.syncRuleProvider.exitOnError,
172
214
  version_label: versionLabel
173
215
  })
174
216
  );
@@ -177,12 +219,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
177
219
  }
178
220
  } catch (e) {
179
221
  // Log and re-raise to exit.
180
- // Should only reach this due to validation errors if exit_on_error is true.
181
222
  this.logger.error(`Failed to update sync config`, e);
182
223
  throw e;
183
224
  }
184
225
  } else {
185
- this.logger.info('No sync streams or rules configured - configure via API');
226
+ this.logger.info('No sync config configured - configure via API');
186
227
  }
187
228
  let useFastRefresh = true;
188
229
  const fastRefreshDeadline = Date.now() + FAST_REFRESH_TIMEOUT_MS;
@@ -290,7 +331,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
290
331
  } else {
291
332
  lock = await replicationStream.lock();
292
333
  }
293
- const syncRuleStorage = this.storage.getInstance(replicationStream);
334
+ const syncRuleStorage = this.storage.getInstance(replicationStream, { replicationLock: lock });
294
335
  const newJob = this.createJob({
295
336
  lock: lock,
296
337
  storage: syncRuleStorage
@@ -63,14 +63,6 @@ export abstract class ReplicationModule<TConfig extends types.configFile.DataSou
63
63
  */
64
64
  protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
65
65
 
66
- /**
67
- * Any additional initialization specific to the module should be added here. Will be called if necessary after the
68
- * main initialization has been completed
69
- * @param context
70
- * @protected
71
- */
72
- protected abstract onInitialized(context: system.ServiceContext): Promise<void>;
73
-
74
66
  public abstract testConnection(config: TConfig): Promise<ConnectionTestResult>;
75
67
 
76
68
  /**
@@ -98,10 +90,18 @@ export abstract class ReplicationModule<TConfig extends types.configFile.DataSou
98
90
  // If decoding fails, this will raise a hard error, and stop the service.
99
91
  this.decodeConfig(baseMatchingConfig);
100
92
 
101
- context.replicationEngine?.register(this.createReplicator(context));
102
- context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
103
-
104
- await this.onInitialized(context);
93
+ const sourceAPI = this.createRouteAPIAdapter();
94
+ // The router exists even in replication-only mode and owns adapter shutdown through the service lifecycle.
95
+ context.routerEngine.registerAPI(sourceAPI);
96
+ if (context.replicationEngine) {
97
+ const replicator = this.createReplicator(context);
98
+ if (sourceAPI.validateSourceCapabilities != null) {
99
+ // KLUDGE: Reuse the adapter validation for filesystem-loaded configs.
100
+ // A shared validation interface could replace this registration through the route adapter in the future.
101
+ replicator.registerSourceCapabilitiesAssertion((config) => api.assertSourceCapabilities(sourceAPI, config));
102
+ }
103
+ context.replicationEngine.register(replicator);
104
+ }
105
105
  }
106
106
 
107
107
  protected decodeConfig(config: TConfig): void {
@@ -8,11 +8,12 @@ import {
8
8
  SqlSyncRules,
9
9
  StaticSchema
10
10
  } from '@powersync/service-sync-rules';
11
- import { internal_routes } from '@powersync/service-types';
11
+ import { internal_routes, ServiceInfo } from '@powersync/service-types';
12
12
 
13
13
  import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
14
14
  import * as api from '../../api/api-index.js';
15
15
  import * as storage from '../../storage/storage-index.js';
16
+ import { POWERSYNC_VERSION } from '../../util/version.js';
16
17
  import { authApi } from '../auth.js';
17
18
  import { routeDefinition } from '../router.js';
18
19
 
@@ -58,19 +59,28 @@ export const diagnostics = routeDefinition({
58
59
  const { context } = payload;
59
60
  const { service_context } = context;
60
61
  const include_content = payload.params.sync_rules_content ?? false;
62
+ const serviceResponse: { service: ServiceInfo } = {
63
+ service: {
64
+ version: POWERSYNC_VERSION
65
+ }
66
+ };
61
67
 
62
68
  const apiHandler = service_context.routerEngine.getAPI();
63
69
 
64
70
  const status = await apiHandler.getConnectionStatus();
65
71
  if (!status) {
66
72
  return internal_routes.DiagnosticsResponse.encode({
73
+ ...serviceResponse,
67
74
  connections: []
68
75
  });
69
76
  }
70
77
 
71
78
  const {
72
- storageEngine: { activeBucketStorage }
79
+ storageEngine: { activeBucketStorage },
80
+ configuration
73
81
  } = service_context;
82
+
83
+ const sync_config_source = configuration?.sync_rules.present ? 'config' : 'api';
74
84
  const active = await activeBucketStorage.getActiveSyncConfig();
75
85
  const deploying = await activeBucketStorage.getDeployingSyncConfig();
76
86
 
@@ -102,6 +112,7 @@ export const diagnostics = routeDefinition({
102
112
  );
103
113
 
104
114
  return internal_routes.DiagnosticsResponse.encode({
115
+ ...serviceResponse,
105
116
  connections: [
106
117
  {
107
118
  ...status,
@@ -110,7 +121,8 @@ export const diagnostics = routeDefinition({
110
121
  }
111
122
  ],
112
123
  active_sync_rules: active_status,
113
- deploying_sync_rules: deploying_status
124
+ deploying_sync_rules: deploying_status,
125
+ sync_config_source
114
126
  });
115
127
  }
116
128
  });
@@ -158,14 +170,21 @@ export const reprocess = routeDefinition({
158
170
  });
159
171
  }
160
172
  // There are some differences between this and using asUpdateOptions():
161
- // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
173
+ // 1. This always re-parses the source YAML. If there are changes to the Sync Streams compiler, that can affect the sync plan.
162
174
  // 2. If the source does not set the storage version, this will update it do the current version.
163
175
  // We can consider tweaking this behavior in the future.
176
+ const parsed = SqlSyncRules.fromYaml(active.content.sync_rules_content, {
177
+ ...apiHandler.getParseSyncRulesOptions(),
178
+ schema: undefined,
179
+ // This sync config already passed validation. But if the config is not valid anymore due
180
+ // to a service change, we do want to report the error here.
181
+ throwOnError: true
182
+ });
183
+ storage.logSyncConfigErrors(parsed);
184
+ await api.assertSourceCapabilities(apiHandler, parsed.config);
185
+
164
186
  const new_rules = await activeBucketStorage.updateSyncRules(
165
- storage.updateSyncRulesFromYaml(active.content.sync_rules_content, {
166
- // This sync config already passed validation. But if the config is not valid anymore due
167
- // to a service change, we do want to report the error here.
168
- validate: true,
187
+ storage.updateSyncRulesFromConfig(parsed, {
169
188
  version_label: active.content.version_label,
170
189
  forceNewReplicationStream: true
171
190
  })
@@ -188,7 +207,6 @@ export const reprocess = routeDefinition({
188
207
 
189
208
  class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigContent {
190
209
  constructor(
191
- private readonly apiHandler: api.RouteAPI,
192
210
  private readonly schema: SourceSchema,
193
211
  data: storage.PersistedSyncConfigContentData
194
212
  ) {
@@ -197,7 +215,7 @@ class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigConte
197
215
 
198
216
  parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
199
217
  const syncConfig = SqlSyncRules.fromYaml(this.sync_rules_content, {
200
- ...this.apiHandler.getParseSyncRulesOptions(),
218
+ ...options,
201
219
  schema: this.schema
202
220
  });
203
221
 
@@ -237,7 +255,7 @@ export const validate = routeDefinition({
237
255
  const schemaData = await api.getConnectionsSchema(apiHandler);
238
256
  const schema = new StaticSchema(schemaData.connections);
239
257
 
240
- const sync_rules = new FakeSyncRulesContentForValidation(apiHandler, schema, {
258
+ const sync_rules = new FakeSyncRulesContentForValidation(schema, {
241
259
  // Dummy values
242
260
  replicationStreamId: 0,
243
261
  replicationStreamName: '',