@powersync/service-core 1.26.1 → 1.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/dist/api/RouteAPI.d.ts +8 -1
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.js +33 -16
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/source-capabilities.d.ts +7 -0
- package/dist/api/source-capabilities.js +23 -0
- package/dist/api/source-capabilities.js.map +1 -0
- package/dist/auth/JwtPayload.d.ts +2 -2
- package/dist/auth/JwtPayload.js +2 -2
- package/dist/auth/JwtPayload.js.map +1 -1
- package/dist/entry/commands/compact-action.js +1 -1
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/config-command.js +1 -1
- package/dist/entry/commands/config-command.js.map +1 -1
- package/dist/migrations/PowerSyncMigrationManager.js +2 -1
- package/dist/migrations/PowerSyncMigrationManager.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +4 -6
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +15 -0
- package/dist/replication/AbstractReplicator.js +37 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +0 -7
- package/dist/replication/ReplicationModule.js +13 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +39 -27
- package/dist/routes/endpoints/admin.d.ts +78 -54
- package/dist/routes/endpoints/admin.js +24 -10
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +7 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +39 -6
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +20 -3
- package/dist/storage/BucketStorageFactory.js +25 -2
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.d.ts +2 -1
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +3 -0
- package/dist/storage/StorageVersionConfig.d.ts +0 -4
- package/dist/storage/StorageVersionConfig.js +0 -4
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +3 -1
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +9 -5
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +1 -1
- package/dist/sync/BucketChecksumState.js +5 -4
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/util/param-logging.d.ts +5 -2
- package/dist/util/param-logging.js +17 -9
- package/dist/util/param-logging.js.map +1 -1
- package/dist/util/protocol-types.d.ts +7 -7
- package/dist/util/protocol-types.js +3 -3
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +9 -1
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +30 -14
- package/src/api/source-capabilities.ts +24 -0
- package/src/auth/JwtPayload.ts +2 -2
- package/src/entry/commands/compact-action.ts +1 -1
- package/src/entry/commands/config-command.ts +1 -1
- package/src/migrations/PowerSyncMigrationManager.ts +2 -1
- package/src/replication/AbstractReplicationJob.ts +2 -4
- package/src/replication/AbstractReplicator.ts +46 -5
- package/src/replication/ReplicationModule.ts +12 -12
- package/src/routes/endpoints/admin.ts +29 -11
- package/src/routes/endpoints/socket-route.ts +9 -6
- package/src/routes/endpoints/sync-rules.ts +42 -9
- package/src/storage/BucketStorageFactory.ts +33 -4
- package/src/storage/PersistedSyncConfigContent.ts +3 -0
- package/src/storage/StorageProvider.ts +3 -0
- package/src/storage/StorageVersionConfig.ts +0 -5
- package/src/storage/SyncRulesBucketStorage.ts +3 -1
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +17 -5
- package/src/sync/BucketChecksumState.ts +5 -4
- package/src/util/param-logging.ts +21 -14
- package/src/util/protocol-types.ts +5 -5
- package/test/src/AbstractReplicator.test.ts +156 -0
- package/test/src/broadcast_iterable.test.ts +26 -8
- package/test/src/diagnostics.test.ts +128 -0
- package/test/src/routes/admin.test.ts +235 -4
- package/test/src/routes/sync-rules.test.ts +124 -0
- package/test/src/source_table_config.test.ts +104 -0
- package/test/src/storage/BucketStorageFactory.test.ts +46 -0
- package/test/src/sync/BucketChecksumState.test.ts +195 -6
- package/test/src/util/param-logging.test.ts +59 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -9,8 +9,9 @@ export const DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS = {
|
|
|
9
9
|
* Formats potentially arbitrary parameters for logging.
|
|
10
10
|
* This limits the number of keys and strings to a maximum length.
|
|
11
11
|
* A warning key-value is added if the number of keys exceeds the maximum.
|
|
12
|
-
*
|
|
12
|
+
* Keys and string values exceeding the maximum length are truncated.
|
|
13
13
|
* Non-String values are stringified, the maximum length is then applied.
|
|
14
|
+
* Values that cannot be serialized are replaced with a placeholder.
|
|
14
15
|
* @param params - The parameters to format.
|
|
15
16
|
* @param options - The options to use.
|
|
16
17
|
* @default DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS
|
|
@@ -24,17 +25,24 @@ export function limitParamsForLogging(params, options = DEFAULT_PARAM_LOGGING_FO
|
|
|
24
25
|
}
|
|
25
26
|
return value;
|
|
26
27
|
}
|
|
27
|
-
|
|
28
|
-
if (
|
|
29
|
-
return
|
|
28
|
+
function formatValue(value) {
|
|
29
|
+
if (typeof value == 'string') {
|
|
30
|
+
return trimString(value);
|
|
30
31
|
}
|
|
31
|
-
|
|
32
|
-
return [];
|
|
32
|
+
try {
|
|
33
|
+
return trimString(JSON.stringify(value) ?? '[undefined]');
|
|
33
34
|
}
|
|
34
|
-
|
|
35
|
-
return
|
|
35
|
+
catch {
|
|
36
|
+
return trimString('[unserializable]');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return Object.fromEntries(Object.entries(params)
|
|
40
|
+
.slice(0, maxKeyCount + 1)
|
|
41
|
+
.map(([key, value], index) => {
|
|
42
|
+
if (index == maxKeyCount) {
|
|
43
|
+
return ['⚠️', 'Additional parameters omitted'];
|
|
36
44
|
}
|
|
37
|
-
return [key,
|
|
45
|
+
return [trimString(key), formatValue(value)];
|
|
38
46
|
}));
|
|
39
47
|
}
|
|
40
48
|
//# sourceMappingURL=param-logging.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"param-logging.js","sourceRoot":"","sources":["../../src/util/param-logging.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAA8B;IAC7E,WAAW,EAAE,EAAE;IACf,eAAe,EAAE,GAAG;CACrB,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"param-logging.js","sourceRoot":"","sources":["../../src/util/param-logging.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAA8B;IAC7E,WAAW,EAAE,EAAE;IACf,eAAe,EAAE,GAAG;CACrB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAA2B,EAC3B,UAA8C,oCAAoC;IAElF,MAAM,EACJ,eAAe,GAAG,oCAAoC,CAAC,eAAe,EACtE,WAAW,GAAG,oCAAoC,CAAC,WAAW,EAC/D,GAAG,OAAO,CAAC;IAEZ,SAAS,UAAU,CAAC,KAAa;QAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,WAAW,CAAC,KAAc;QACjC,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC;YACH,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACnB,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;QAC3B,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,EAAE,+BAA+B,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CACL,CAAC;AACJ,CAAC"}
|
|
@@ -14,7 +14,7 @@ export type BucketRequest = t.Decoded<typeof BucketRequest>;
|
|
|
14
14
|
*/
|
|
15
15
|
export declare const RequestedStreamSubscription: t.ObjectCodec<{
|
|
16
16
|
/**
|
|
17
|
-
* The defined name of the stream as it appears in
|
|
17
|
+
* The defined name of the stream as it appears in Sync Stream definitions.
|
|
18
18
|
*/
|
|
19
19
|
stream: t.IdentityCodec<t.CodecType.String>;
|
|
20
20
|
/**
|
|
@@ -24,7 +24,7 @@ export declare const RequestedStreamSubscription: t.ObjectCodec<{
|
|
|
24
24
|
/**
|
|
25
25
|
* Set when the client wishes to re-assign a different priority to this stream.
|
|
26
26
|
*
|
|
27
|
-
* Streams and
|
|
27
|
+
* Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
|
|
28
28
|
* useful when the priority for partial syncs depends on e.g. the current page opened in a client.
|
|
29
29
|
*/
|
|
30
30
|
override_priority: t.Codec<number | null, number | null>;
|
|
@@ -41,11 +41,11 @@ export declare const StreamSubscriptionRequest: t.ObjectCodec<{
|
|
|
41
41
|
*/
|
|
42
42
|
include_defaults: t.OptionalCodec<t.Codec<boolean, boolean, string, t.CodecProps>>;
|
|
43
43
|
/**
|
|
44
|
-
* An array of
|
|
44
|
+
* An array of Sync Streams the client has opened explicitly.
|
|
45
45
|
*/
|
|
46
46
|
subscriptions: t.ArrayCodec<t.ObjectCodec<{
|
|
47
47
|
/**
|
|
48
|
-
* The defined name of the stream as it appears in
|
|
48
|
+
* The defined name of the stream as it appears in Sync Stream definitions.
|
|
49
49
|
*/
|
|
50
50
|
stream: t.IdentityCodec<t.CodecType.String>;
|
|
51
51
|
/**
|
|
@@ -55,7 +55,7 @@ export declare const StreamSubscriptionRequest: t.ObjectCodec<{
|
|
|
55
55
|
/**
|
|
56
56
|
* Set when the client wishes to re-assign a different priority to this stream.
|
|
57
57
|
*
|
|
58
|
-
* Streams and
|
|
58
|
+
* Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
|
|
59
59
|
* useful when the priority for partial syncs depends on e.g. the current page opened in a client.
|
|
60
60
|
*/
|
|
61
61
|
override_priority: t.Codec<number | null, number | null>;
|
|
@@ -239,8 +239,8 @@ export interface BucketChecksum {
|
|
|
239
239
|
/**
|
|
240
240
|
* The reason a particular bucket is included in a checkpoint.
|
|
241
241
|
*
|
|
242
|
-
* This information allows clients to associate individual buckets with
|
|
243
|
-
* association is useful because it enables clients to track progress for individual
|
|
242
|
+
* This information allows clients to associate individual buckets with Sync Streams they're subscribed to. Having that
|
|
243
|
+
* association is useful because it enables clients to track progress for individual Sync Streams.
|
|
244
244
|
*/
|
|
245
245
|
export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
|
|
246
246
|
/**
|
|
@@ -12,7 +12,7 @@ export const BucketRequest = t.object({
|
|
|
12
12
|
*/
|
|
13
13
|
export const RequestedStreamSubscription = t.object({
|
|
14
14
|
/**
|
|
15
|
-
* The defined name of the stream as it appears in
|
|
15
|
+
* The defined name of the stream as it appears in Sync Stream definitions.
|
|
16
16
|
*/
|
|
17
17
|
stream: t.string,
|
|
18
18
|
/**
|
|
@@ -22,7 +22,7 @@ export const RequestedStreamSubscription = t.object({
|
|
|
22
22
|
/**
|
|
23
23
|
* Set when the client wishes to re-assign a different priority to this stream.
|
|
24
24
|
*
|
|
25
|
-
* Streams and
|
|
25
|
+
* Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
|
|
26
26
|
* useful when the priority for partial syncs depends on e.g. the current page opened in a client.
|
|
27
27
|
*/
|
|
28
28
|
override_priority: orNull(t.number)
|
|
@@ -38,7 +38,7 @@ export const StreamSubscriptionRequest = t.object({
|
|
|
38
38
|
*/
|
|
39
39
|
include_defaults: t.boolean.optional(),
|
|
40
40
|
/**
|
|
41
|
-
* An array of
|
|
41
|
+
* An array of Sync Streams the client has opened explicitly.
|
|
42
42
|
*/
|
|
43
43
|
subscriptions: t.array(RequestedStreamSubscription)
|
|
44
44
|
});
|
package/package.json
CHANGED
|
@@ -4,18 +4,18 @@
|
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
7
|
-
"version": "1.
|
|
7
|
+
"version": "1.27.1",
|
|
8
8
|
"license": "FSL-1.1-ALv2",
|
|
9
9
|
"type": "module",
|
|
10
10
|
"exports": "./dist/index.js",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@js-sdsl/ordered-set": "^4.4.2",
|
|
13
13
|
"@opentelemetry/api": "^1.9.0",
|
|
14
|
-
"@opentelemetry/core": "^2.0
|
|
15
|
-
"@opentelemetry/exporter-metrics-otlp-http": "^0.
|
|
16
|
-
"@opentelemetry/exporter-prometheus": "^0.
|
|
17
|
-
"@opentelemetry/resources": "^2.0
|
|
18
|
-
"@opentelemetry/sdk-metrics": "^2.0
|
|
14
|
+
"@opentelemetry/core": "^2.8.0",
|
|
15
|
+
"@opentelemetry/exporter-metrics-otlp-http": "^0.219.0",
|
|
16
|
+
"@opentelemetry/exporter-prometheus": "^0.219.0",
|
|
17
|
+
"@opentelemetry/resources": "^2.8.0",
|
|
18
|
+
"@opentelemetry/sdk-metrics": "^2.8.0",
|
|
19
19
|
"async": "^3.2.4",
|
|
20
20
|
"async-mutex": "^0.5.0",
|
|
21
21
|
"bson": "^6.10.4",
|
|
@@ -33,17 +33,17 @@
|
|
|
33
33
|
"uuid": "^14.0.0",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.8.3",
|
|
36
|
-
"@powersync/
|
|
36
|
+
"@powersync/service-rsocket-router": "0.2.29",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.13",
|
|
38
|
-
"@powersync/service-
|
|
39
|
-
"@powersync/service-
|
|
40
|
-
"@powersync/
|
|
38
|
+
"@powersync/service-types": "0.20.0",
|
|
39
|
+
"@powersync/service-sync-rules": "0.44.0",
|
|
40
|
+
"@powersync/lib-services-framework": "0.10.4"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
|
44
44
|
"@types/lodash": "^4.17.5",
|
|
45
45
|
"@types/negotiator": "^0.6.4",
|
|
46
|
-
"fastify": "^5.
|
|
46
|
+
"fastify": "^5.12.5",
|
|
47
47
|
"fastify-plugin": "^5.0.1"
|
|
48
48
|
},
|
|
49
49
|
"scripts": {
|
package/src/api/RouteAPI.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as types from '@powersync/service-types';
|
|
3
|
-
import { ParseSyncConfigOptions
|
|
3
|
+
import type { ParseSyncConfigOptions } from '../storage/PersistedSyncConfigContent.js';
|
|
4
|
+
import { SyncRulesBucketStorage } from '../storage/storage-index.js';
|
|
4
5
|
|
|
5
6
|
export interface PatternResult {
|
|
6
7
|
schema: string;
|
|
@@ -40,6 +41,13 @@ export interface RouteAPI {
|
|
|
40
41
|
*/
|
|
41
42
|
getConnectionStatus(): Promise<types.ConnectionStatusV2>;
|
|
42
43
|
|
|
44
|
+
/**
|
|
45
|
+
* Checks whether the source supports the capabilities required by this sync config.
|
|
46
|
+
* Must not modify the source. Return advisory warnings and fatal validation diagnostics.
|
|
47
|
+
* An empty array means no issues; throw only for unexpected failures.
|
|
48
|
+
*/
|
|
49
|
+
validateSourceCapabilities?(syncConfig: SyncConfig): Promise<types.ValidationDiagnostic[]>;
|
|
50
|
+
|
|
43
51
|
/**
|
|
44
52
|
* Expand sync config table patterns into table metadata from the source connection.
|
|
45
53
|
*
|
package/src/api/api-index.ts
CHANGED
package/src/api/diagnostics.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
-
import { DEFAULT_TAG, SourceTableRef, SyncConfigWithErrors } from '@powersync/service-sync-rules';
|
|
2
|
+
import { DEFAULT_TAG, SourceTableRef, SyncConfigWithErrors, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
3
3
|
import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types';
|
|
4
4
|
|
|
5
5
|
import * as storage from '../storage/storage-index.js';
|
|
@@ -68,7 +68,11 @@ export async function getSyncRulesStatus(
|
|
|
68
68
|
version_label: syncConfig.version_label,
|
|
69
69
|
content: include_content ? syncConfig.sync_rules_content : undefined,
|
|
70
70
|
connections: [],
|
|
71
|
-
|
|
71
|
+
// Report each sync config error separately, retaining its source location.
|
|
72
|
+
errors:
|
|
73
|
+
e instanceof SyncRulesErrors
|
|
74
|
+
? e.errors.map((error) => syncConfigYamlErrorToReplicationError(error, now))
|
|
75
|
+
: [{ level: 'fatal', message: e.message, ts: now }]
|
|
72
76
|
};
|
|
73
77
|
}
|
|
74
78
|
|
|
@@ -81,20 +85,32 @@ export async function getSyncRulesStatus(
|
|
|
81
85
|
let replication_lag_bytes: number | undefined = undefined;
|
|
82
86
|
let slot_wal_budget: SlotWalBudgetInfo | undefined = undefined;
|
|
83
87
|
|
|
88
|
+
const validationErrors: ReplicationError[] = [];
|
|
84
89
|
let tables_flat: TableInfo[] = [];
|
|
85
90
|
|
|
86
91
|
if (check_connection) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
92
|
+
try {
|
|
93
|
+
validationErrors.push(...((await apiHandler.validateSourceCapabilities?.(rules)) ?? []));
|
|
94
|
+
} catch (e) {
|
|
95
|
+
validationErrors.push({ level: 'fatal', message: e.message, ts: now });
|
|
96
|
+
}
|
|
97
|
+
// Table inspection is independent of source capability checks. Preserve both failures
|
|
98
|
+
// and continue collecting diagnostics that do not depend on table metadata.
|
|
99
|
+
try {
|
|
100
|
+
const source_table_patterns = rules.getSourceTables();
|
|
101
|
+
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
102
|
+
tables_flat = resolved_tables.flatMap((info) => {
|
|
103
|
+
if (info.table) {
|
|
104
|
+
return [info.table];
|
|
105
|
+
} else if (info.tables) {
|
|
106
|
+
return info.tables;
|
|
107
|
+
} else {
|
|
108
|
+
return [];
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
} catch (e) {
|
|
112
|
+
validationErrors.push({ level: 'fatal', message: e.message, ts: now });
|
|
113
|
+
}
|
|
98
114
|
|
|
99
115
|
if (systemStorage) {
|
|
100
116
|
try {
|
|
@@ -151,7 +167,7 @@ export async function getSyncRulesStatus(
|
|
|
151
167
|
});
|
|
152
168
|
}
|
|
153
169
|
|
|
154
|
-
const errors = tables_flat.flatMap((info) => info.errors);
|
|
170
|
+
const errors = [...validationErrors, ...tables_flat.flatMap((info) => info.errors)];
|
|
155
171
|
const statusSource = await syncConfig.getSyncConfigStatus();
|
|
156
172
|
|
|
157
173
|
if (statusSource?.last_fatal_error) {
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import type { SyncConfig } from '@powersync/service-sync-rules';
|
|
3
|
+
import type { RouteAPI } from './RouteAPI.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Blocks deployment and reprocessing on fatal source diagnostics; warnings remain advisory.
|
|
7
|
+
* Capability failures use the same typed 422 error for deployment and reprocessing.
|
|
8
|
+
*/
|
|
9
|
+
export async function assertSourceCapabilities(api: RouteAPI, config: SyncConfig): Promise<void> {
|
|
10
|
+
try {
|
|
11
|
+
const diagnostics = (await api.validateSourceCapabilities?.(config)) ?? [];
|
|
12
|
+
const errors = diagnostics.filter((diagnostic) => diagnostic.level === 'fatal');
|
|
13
|
+
if (errors.length > 0) {
|
|
14
|
+
throw new Error(errors.map((error) => error.message).join('\n'));
|
|
15
|
+
}
|
|
16
|
+
} catch (error) {
|
|
17
|
+
throw new ServiceError({
|
|
18
|
+
status: 422,
|
|
19
|
+
code: ErrorCode.PSYNC_R0001,
|
|
20
|
+
description: 'Source capability validation failed',
|
|
21
|
+
details: error.message
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
}
|
package/src/auth/JwtPayload.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RequestJwtPayload } from '@powersync/service-sync-rules';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Payload from a JWT, always signed.
|
|
5
5
|
*/
|
|
6
|
-
export class JwtPayload extends
|
|
6
|
+
export class JwtPayload extends RequestJwtPayload {
|
|
7
7
|
/**
|
|
8
8
|
* Stringified version of sub. Used where we need a string user identifier, such as in write checkpoints
|
|
9
9
|
* and per-user metrics.
|
|
@@ -46,7 +46,7 @@ export function registerCompactAction(program: Command) {
|
|
|
46
46
|
if (invalid.length > 0) {
|
|
47
47
|
logger.error(
|
|
48
48
|
`Invalid bucket names: ${invalid.join(', ')}. ` +
|
|
49
|
-
`Pass full bucket names (e.g., "global[]"), not bucket definition names (e.g., "global").`
|
|
49
|
+
`Pass full bucket names (e.g., "global[]"), not bucket definition or Sync Stream names (e.g., "global").`
|
|
50
50
|
);
|
|
51
51
|
process.exit(1);
|
|
52
52
|
}
|
|
@@ -19,7 +19,7 @@ export function wrapConfigCommand(command: Command) {
|
|
|
19
19
|
)
|
|
20
20
|
.option(
|
|
21
21
|
`-sync64, --sync-base64 [base64]`,
|
|
22
|
-
'Base64 encoded YAML Sync Config. Defaults to process.env.POWERSYNC_SYNC_CONFIG_B64 or process.env.POWERSYNC_SYNC_RULES_B64 for backwards
|
|
22
|
+
'Base64 encoded YAML Sync Config. Defaults to process.env.POWERSYNC_SYNC_CONFIG_B64 or process.env.POWERSYNC_SYNC_RULES_B64 for backwards compatibility.',
|
|
23
23
|
util.env.POWERSYNC_SYNC_CONFIG_B64 || util.env.POWERSYNC_SYNC_RULES_B64
|
|
24
24
|
);
|
|
25
25
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as framework from '@powersync/lib-services-framework';
|
|
2
2
|
import fs from 'fs/promises';
|
|
3
3
|
import path from 'path';
|
|
4
|
+
import { pathToFileURL } from 'url';
|
|
4
5
|
import * as system from '../system/system-index.js';
|
|
5
6
|
|
|
6
7
|
/**
|
|
@@ -28,7 +29,7 @@ export abstract class AbstractPowerSyncMigrationAgent extends framework.Abstract
|
|
|
28
29
|
|
|
29
30
|
return await Promise.all(
|
|
30
31
|
migrations.map(async (migration) => {
|
|
31
|
-
const module = await import(path.resolve(migrationsDir, migration));
|
|
32
|
+
const module = await import(pathToFileURL(path.resolve(migrationsDir, migration)).href);
|
|
32
33
|
return {
|
|
33
34
|
name: path.basename(migration).replace(path.extname(migration), ''),
|
|
34
35
|
up: module.up,
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { container, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import winston from 'winston';
|
|
3
3
|
import { MetricsEngine } from '../metrics/MetricsEngine.js';
|
|
4
|
-
import
|
|
4
|
+
import type { CreateJobOptions } from './AbstractReplicator.js';
|
|
5
5
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
6
6
|
|
|
7
|
-
export interface AbstractReplicationJobOptions {
|
|
7
|
+
export interface AbstractReplicationJobOptions extends CreateJobOptions {
|
|
8
8
|
id: string;
|
|
9
|
-
storage: storage.SyncRulesBucketStorage;
|
|
10
9
|
metrics: MetricsEngine;
|
|
11
|
-
lock: storage.ReplicationLock;
|
|
12
10
|
rateLimiter: ErrorRateLimiter;
|
|
13
11
|
}
|
|
14
12
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { container, ErrorCode, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { DEFAULT_TAG, SqlSyncRules, type SyncConfig } from '@powersync/service-sync-rules';
|
|
2
3
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
3
4
|
import { hrtime } from 'node:process';
|
|
4
5
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
@@ -7,6 +8,7 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js';
|
|
|
7
8
|
import * as storage from '../storage/storage-index.js';
|
|
8
9
|
import { StorageEngine } from '../storage/storage-index.js';
|
|
9
10
|
import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
|
|
11
|
+
import type { ResolvedPowerSyncConfig } from '../util/config/types.js';
|
|
10
12
|
import { AbstractReplicationJob } from './AbstractReplicationJob.js';
|
|
11
13
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
12
14
|
import { ConnectionTestResult } from './ReplicationModule.js';
|
|
@@ -30,6 +32,10 @@ export interface AbstractReplicatorOptions {
|
|
|
30
32
|
storageEngine: StorageEngine;
|
|
31
33
|
metricsEngine: MetricsEngine;
|
|
32
34
|
syncRuleProvider: SyncRulesProvider;
|
|
35
|
+
/**
|
|
36
|
+
* Source connection metadata from the service config, used when parsing filesystem-loaded sync configs.
|
|
37
|
+
*/
|
|
38
|
+
configuration?: Pick<ResolvedPowerSyncConfig, 'connections'>;
|
|
33
39
|
/**
|
|
34
40
|
* This limits the effect of retries when there is a persistent issue.
|
|
35
41
|
*/
|
|
@@ -73,6 +79,8 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
73
79
|
|
|
74
80
|
private abortController: AbortController | undefined;
|
|
75
81
|
|
|
82
|
+
private assertSourceCapabilities: ((config: SyncConfig) => Promise<void>) | undefined;
|
|
83
|
+
|
|
76
84
|
protected constructor(private options: AbstractReplicatorOptions) {
|
|
77
85
|
this.logger = logger.child({ name: `Replicator:${options.id}` });
|
|
78
86
|
const heartbeatIntervalSeconds = options.heartbeatIntervalSeconds ?? DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
|
|
@@ -105,6 +113,13 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
105
113
|
return this.options.syncRuleProvider;
|
|
106
114
|
}
|
|
107
115
|
|
|
116
|
+
/**
|
|
117
|
+
* Default schema for startup sync config validation. Source modules with a known default should override this.
|
|
118
|
+
*/
|
|
119
|
+
protected get defaultSchema(): string {
|
|
120
|
+
return 'not_applicable';
|
|
121
|
+
}
|
|
122
|
+
|
|
108
123
|
protected get rateLimiter() {
|
|
109
124
|
return this.options.rateLimiter;
|
|
110
125
|
}
|
|
@@ -117,6 +132,13 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
117
132
|
return this.abortController?.signal.aborted;
|
|
118
133
|
}
|
|
119
134
|
|
|
135
|
+
/**
|
|
136
|
+
* Register an assertion that blocks unsupported file-loaded sync configs before replication.
|
|
137
|
+
*/
|
|
138
|
+
public registerSourceCapabilitiesAssertion(assertion: (config: SyncConfig) => Promise<void>): void {
|
|
139
|
+
this.assertSourceCapabilities = assertion;
|
|
140
|
+
}
|
|
141
|
+
|
|
120
142
|
public async start(): Promise<void> {
|
|
121
143
|
this.abortController = new AbortController();
|
|
122
144
|
this.runLoop().catch((e) => {
|
|
@@ -165,10 +187,30 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
165
187
|
// versions before that check was added, so we keep the lock for now - for where te service version and sync config is updated at
|
|
166
188
|
// the same time.
|
|
167
189
|
|
|
190
|
+
const parsed = SqlSyncRules.fromYaml(loadedSyncConfig, {
|
|
191
|
+
schema: undefined,
|
|
192
|
+
defaultSchema: this.defaultSchema,
|
|
193
|
+
connections:
|
|
194
|
+
this.options.configuration?.connections &&
|
|
195
|
+
Object.fromEntries(
|
|
196
|
+
this.options.configuration.connections.map((connection) => [
|
|
197
|
+
connection.tag ?? DEFAULT_TAG,
|
|
198
|
+
{ type: connection.type }
|
|
199
|
+
])
|
|
200
|
+
),
|
|
201
|
+
throwOnError: this.syncRuleProvider.exitOnError
|
|
202
|
+
});
|
|
203
|
+
storage.logSyncConfigErrors(parsed, this.logger);
|
|
204
|
+
try {
|
|
205
|
+
await this.assertSourceCapabilities?.(parsed.config);
|
|
206
|
+
} catch (error) {
|
|
207
|
+
if (this.syncRuleProvider.exitOnError) throw error;
|
|
208
|
+
// Persist the config for diagnostics; the job-start check will prevent unsafe replication.
|
|
209
|
+
this.logger.error('Source capability validation failed for the sync config', error);
|
|
210
|
+
}
|
|
168
211
|
const { lock } = await this.storage.configureSyncRules(
|
|
169
|
-
storage.
|
|
212
|
+
storage.updateSyncRulesFromConfig(parsed, {
|
|
170
213
|
lock: true,
|
|
171
|
-
validate: this.syncRuleProvider.exitOnError,
|
|
172
214
|
version_label: versionLabel
|
|
173
215
|
})
|
|
174
216
|
);
|
|
@@ -177,12 +219,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
177
219
|
}
|
|
178
220
|
} catch (e) {
|
|
179
221
|
// Log and re-raise to exit.
|
|
180
|
-
// Should only reach this due to validation errors if exit_on_error is true.
|
|
181
222
|
this.logger.error(`Failed to update sync config`, e);
|
|
182
223
|
throw e;
|
|
183
224
|
}
|
|
184
225
|
} else {
|
|
185
|
-
this.logger.info('No sync
|
|
226
|
+
this.logger.info('No sync config configured - configure via API');
|
|
186
227
|
}
|
|
187
228
|
let useFastRefresh = true;
|
|
188
229
|
const fastRefreshDeadline = Date.now() + FAST_REFRESH_TIMEOUT_MS;
|
|
@@ -290,7 +331,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
290
331
|
} else {
|
|
291
332
|
lock = await replicationStream.lock();
|
|
292
333
|
}
|
|
293
|
-
const syncRuleStorage = this.storage.getInstance(replicationStream);
|
|
334
|
+
const syncRuleStorage = this.storage.getInstance(replicationStream, { replicationLock: lock });
|
|
294
335
|
const newJob = this.createJob({
|
|
295
336
|
lock: lock,
|
|
296
337
|
storage: syncRuleStorage
|
|
@@ -63,14 +63,6 @@ export abstract class ReplicationModule<TConfig extends types.configFile.DataSou
|
|
|
63
63
|
*/
|
|
64
64
|
protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
|
|
65
65
|
|
|
66
|
-
/**
|
|
67
|
-
* Any additional initialization specific to the module should be added here. Will be called if necessary after the
|
|
68
|
-
* main initialization has been completed
|
|
69
|
-
* @param context
|
|
70
|
-
* @protected
|
|
71
|
-
*/
|
|
72
|
-
protected abstract onInitialized(context: system.ServiceContext): Promise<void>;
|
|
73
|
-
|
|
74
66
|
public abstract testConnection(config: TConfig): Promise<ConnectionTestResult>;
|
|
75
67
|
|
|
76
68
|
/**
|
|
@@ -98,10 +90,18 @@ export abstract class ReplicationModule<TConfig extends types.configFile.DataSou
|
|
|
98
90
|
// If decoding fails, this will raise a hard error, and stop the service.
|
|
99
91
|
this.decodeConfig(baseMatchingConfig);
|
|
100
92
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
93
|
+
const sourceAPI = this.createRouteAPIAdapter();
|
|
94
|
+
// The router exists even in replication-only mode and owns adapter shutdown through the service lifecycle.
|
|
95
|
+
context.routerEngine.registerAPI(sourceAPI);
|
|
96
|
+
if (context.replicationEngine) {
|
|
97
|
+
const replicator = this.createReplicator(context);
|
|
98
|
+
if (sourceAPI.validateSourceCapabilities != null) {
|
|
99
|
+
// KLUDGE: Reuse the adapter validation for filesystem-loaded configs.
|
|
100
|
+
// A shared validation interface could replace this registration through the route adapter in the future.
|
|
101
|
+
replicator.registerSourceCapabilitiesAssertion((config) => api.assertSourceCapabilities(sourceAPI, config));
|
|
102
|
+
}
|
|
103
|
+
context.replicationEngine.register(replicator);
|
|
104
|
+
}
|
|
105
105
|
}
|
|
106
106
|
|
|
107
107
|
protected decodeConfig(config: TConfig): void {
|
|
@@ -8,11 +8,12 @@ import {
|
|
|
8
8
|
SqlSyncRules,
|
|
9
9
|
StaticSchema
|
|
10
10
|
} from '@powersync/service-sync-rules';
|
|
11
|
-
import { internal_routes } from '@powersync/service-types';
|
|
11
|
+
import { internal_routes, ServiceInfo } from '@powersync/service-types';
|
|
12
12
|
|
|
13
13
|
import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
|
|
14
14
|
import * as api from '../../api/api-index.js';
|
|
15
15
|
import * as storage from '../../storage/storage-index.js';
|
|
16
|
+
import { POWERSYNC_VERSION } from '../../util/version.js';
|
|
16
17
|
import { authApi } from '../auth.js';
|
|
17
18
|
import { routeDefinition } from '../router.js';
|
|
18
19
|
|
|
@@ -58,19 +59,28 @@ export const diagnostics = routeDefinition({
|
|
|
58
59
|
const { context } = payload;
|
|
59
60
|
const { service_context } = context;
|
|
60
61
|
const include_content = payload.params.sync_rules_content ?? false;
|
|
62
|
+
const serviceResponse: { service: ServiceInfo } = {
|
|
63
|
+
service: {
|
|
64
|
+
version: POWERSYNC_VERSION
|
|
65
|
+
}
|
|
66
|
+
};
|
|
61
67
|
|
|
62
68
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
63
69
|
|
|
64
70
|
const status = await apiHandler.getConnectionStatus();
|
|
65
71
|
if (!status) {
|
|
66
72
|
return internal_routes.DiagnosticsResponse.encode({
|
|
73
|
+
...serviceResponse,
|
|
67
74
|
connections: []
|
|
68
75
|
});
|
|
69
76
|
}
|
|
70
77
|
|
|
71
78
|
const {
|
|
72
|
-
storageEngine: { activeBucketStorage }
|
|
79
|
+
storageEngine: { activeBucketStorage },
|
|
80
|
+
configuration
|
|
73
81
|
} = service_context;
|
|
82
|
+
|
|
83
|
+
const sync_config_source = configuration?.sync_rules.present ? 'config' : 'api';
|
|
74
84
|
const active = await activeBucketStorage.getActiveSyncConfig();
|
|
75
85
|
const deploying = await activeBucketStorage.getDeployingSyncConfig();
|
|
76
86
|
|
|
@@ -102,6 +112,7 @@ export const diagnostics = routeDefinition({
|
|
|
102
112
|
);
|
|
103
113
|
|
|
104
114
|
return internal_routes.DiagnosticsResponse.encode({
|
|
115
|
+
...serviceResponse,
|
|
105
116
|
connections: [
|
|
106
117
|
{
|
|
107
118
|
...status,
|
|
@@ -110,7 +121,8 @@ export const diagnostics = routeDefinition({
|
|
|
110
121
|
}
|
|
111
122
|
],
|
|
112
123
|
active_sync_rules: active_status,
|
|
113
|
-
deploying_sync_rules: deploying_status
|
|
124
|
+
deploying_sync_rules: deploying_status,
|
|
125
|
+
sync_config_source
|
|
114
126
|
});
|
|
115
127
|
}
|
|
116
128
|
});
|
|
@@ -158,14 +170,21 @@ export const reprocess = routeDefinition({
|
|
|
158
170
|
});
|
|
159
171
|
}
|
|
160
172
|
// There are some differences between this and using asUpdateOptions():
|
|
161
|
-
// 1. This always re-parses the source YAML. If there are changes to the
|
|
173
|
+
// 1. This always re-parses the source YAML. If there are changes to the Sync Streams compiler, that can affect the sync plan.
|
|
162
174
|
// 2. If the source does not set the storage version, this will update it do the current version.
|
|
163
175
|
// We can consider tweaking this behavior in the future.
|
|
176
|
+
const parsed = SqlSyncRules.fromYaml(active.content.sync_rules_content, {
|
|
177
|
+
...apiHandler.getParseSyncRulesOptions(),
|
|
178
|
+
schema: undefined,
|
|
179
|
+
// This sync config already passed validation. But if the config is not valid anymore due
|
|
180
|
+
// to a service change, we do want to report the error here.
|
|
181
|
+
throwOnError: true
|
|
182
|
+
});
|
|
183
|
+
storage.logSyncConfigErrors(parsed);
|
|
184
|
+
await api.assertSourceCapabilities(apiHandler, parsed.config);
|
|
185
|
+
|
|
164
186
|
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
165
|
-
storage.
|
|
166
|
-
// This sync config already passed validation. But if the config is not valid anymore due
|
|
167
|
-
// to a service change, we do want to report the error here.
|
|
168
|
-
validate: true,
|
|
187
|
+
storage.updateSyncRulesFromConfig(parsed, {
|
|
169
188
|
version_label: active.content.version_label,
|
|
170
189
|
forceNewReplicationStream: true
|
|
171
190
|
})
|
|
@@ -188,7 +207,6 @@ export const reprocess = routeDefinition({
|
|
|
188
207
|
|
|
189
208
|
class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigContent {
|
|
190
209
|
constructor(
|
|
191
|
-
private readonly apiHandler: api.RouteAPI,
|
|
192
210
|
private readonly schema: SourceSchema,
|
|
193
211
|
data: storage.PersistedSyncConfigContentData
|
|
194
212
|
) {
|
|
@@ -197,7 +215,7 @@ class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigConte
|
|
|
197
215
|
|
|
198
216
|
parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
199
217
|
const syncConfig = SqlSyncRules.fromYaml(this.sync_rules_content, {
|
|
200
|
-
...
|
|
218
|
+
...options,
|
|
201
219
|
schema: this.schema
|
|
202
220
|
});
|
|
203
221
|
|
|
@@ -237,7 +255,7 @@ export const validate = routeDefinition({
|
|
|
237
255
|
const schemaData = await api.getConnectionsSchema(apiHandler);
|
|
238
256
|
const schema = new StaticSchema(schemaData.connections);
|
|
239
257
|
|
|
240
|
-
const sync_rules = new FakeSyncRulesContentForValidation(
|
|
258
|
+
const sync_rules = new FakeSyncRulesContentForValidation(schema, {
|
|
241
259
|
// Dummy values
|
|
242
260
|
replicationStreamId: 0,
|
|
243
261
|
replicationStreamName: '',
|