@powersync/service-core 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -8
- package/dist/api/RouteAPI.d.ts +8 -0
- package/dist/auth/CachedKeyCollector.js +26 -25
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js +1 -0
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeySpec.js +3 -0
- package/dist/auth/KeySpec.js.map +1 -1
- package/dist/auth/KeyStore.js +4 -0
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/LeakyBucket.js +5 -0
- package/dist/auth/LeakyBucket.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.d.ts +3 -0
- package/dist/auth/RemoteJWKSCollector.js +12 -5
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.js +1 -0
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js +1 -0
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/entry/cli-entry.js +2 -0
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -1
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/commands/test-connection-action.d.ts +2 -0
- package/dist/entry/commands/test-connection-action.js +32 -0
- package/dist/entry/commands/test-connection-action.js.map +1 -0
- package/dist/metrics/Metrics.js +37 -3
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/modules/AbstractModule.js +2 -0
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/ModuleManager.js +1 -3
- package/dist/modules/ModuleManager.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.js +4 -2
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +2 -0
- package/dist/replication/AbstractReplicator.js +18 -12
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationEngine.d.ts +2 -0
- package/dist/replication/ReplicationEngine.js +4 -3
- package/dist/replication/ReplicationEngine.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +8 -2
- package/dist/replication/ReplicationModule.js +9 -11
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/RouterEngine.js +8 -0
- package/dist/routes/RouterEngine.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +3 -3
- package/dist/routes/endpoints/admin.d.ts +6 -6
- package/dist/routes/endpoints/admin.js +7 -4
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +14 -84
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +5 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +16 -11
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +5 -5
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.js +4 -4
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +2 -2
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +32 -5
- package/dist/storage/BucketStorage.js +3 -0
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/ChecksumCache.js +12 -7
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/SourceTable.js +32 -25
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.js +4 -3
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/bson.d.ts +5 -3
- package/dist/storage/bson.js.map +1 -1
- package/dist/sync/BroadcastIterable.js +4 -3
- package/dist/sync/BroadcastIterable.js.map +1 -1
- package/dist/sync/LastValueSink.js +2 -0
- package/dist/sync/LastValueSink.js.map +1 -1
- package/dist/sync/RequestTracker.js +2 -4
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/merge.js +4 -0
- package/dist/sync/merge.js.map +1 -1
- package/dist/sync/sync.js +2 -2
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js +2 -2
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.js +3 -0
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/util/Mutex.js +5 -0
- package/dist/util/Mutex.js.map +1 -1
- package/dist/util/checkpointing.d.ts +13 -0
- package/dist/util/checkpointing.js +92 -0
- package/dist/util/checkpointing.js.map +1 -0
- package/dist/util/config/compound-config-collector.js +3 -1
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +4 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/dist/util/memory-tracking.js +1 -1
- package/dist/util/memory-tracking.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +0 -1
- package/dist/util/utils.js +2 -10
- package/dist/util/utils.js.map +1 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +10 -0
- package/src/auth/RemoteJWKSCollector.ts +18 -5
- package/src/entry/cli-entry.ts +2 -0
- package/src/entry/commands/teardown-action.ts +2 -1
- package/src/entry/commands/test-connection-action.ts +41 -0
- package/src/metrics/Metrics.ts +2 -2
- package/src/replication/AbstractReplicator.ts +12 -3
- package/src/replication/ReplicationEngine.ts +5 -0
- package/src/replication/ReplicationModule.ts +15 -13
- package/src/routes/endpoints/admin.ts +7 -4
- package/src/routes/endpoints/checkpointing.ts +8 -19
- package/src/routes/endpoints/socket-route.ts +5 -5
- package/src/routes/endpoints/sync-rules.ts +16 -11
- package/src/routes/endpoints/sync-stream.ts +5 -5
- package/src/routes/route-register.ts +4 -4
- package/src/storage/BucketStorage.ts +39 -4
- package/src/storage/bson.ts +9 -7
- package/src/util/checkpointing.ts +43 -0
- package/src/util/config/compound-config-collector.ts +2 -1
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +6 -0
- package/src/util/config/types.ts +1 -0
- package/src/util/memory-tracking.ts +2 -2
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +2 -11
- package/tsconfig.tsbuildinfo +1 -1
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20250214100224",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
"uuid": "^9.0.1",
|
|
33
33
|
"winston": "^3.13.0",
|
|
34
34
|
"yaml": "^2.3.2",
|
|
35
|
-
"@powersync/lib-services-framework": "0.
|
|
35
|
+
"@powersync/lib-services-framework": "0.5.1",
|
|
36
36
|
"@powersync/service-jsonbig": "0.17.10",
|
|
37
|
-
"@powersync/service-rsocket-router": "0.0.
|
|
38
|
-
"@powersync/service-sync-rules": "0.23.
|
|
39
|
-
"@powersync/service-types": "0.
|
|
37
|
+
"@powersync/service-rsocket-router": "0.0.18",
|
|
38
|
+
"@powersync/service-sync-rules": "0.23.4",
|
|
39
|
+
"@powersync/service-types": "0.8.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@types/async": "^3.2.24",
|
package/src/api/RouteAPI.ts
CHANGED
|
@@ -54,6 +54,14 @@ export interface RouteAPI {
|
|
|
54
54
|
*/
|
|
55
55
|
getReplicationHead(): Promise<string>;
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Get the current LSN or equivalent replication HEAD position identifier.
|
|
59
|
+
*
|
|
60
|
+
* The position is provided to the callback. After the callback returns,
|
|
61
|
+
* the replication head or a greater one will be streamed on the replication stream.
|
|
62
|
+
*/
|
|
63
|
+
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
64
|
+
|
|
57
65
|
/**
|
|
58
66
|
* @returns The schema for tables inside the connected database. This is typically
|
|
59
67
|
* used to validate sync rules.
|
|
@@ -76,3 +84,5 @@ export interface RouteAPI {
|
|
|
76
84
|
*/
|
|
77
85
|
getParseSyncRulesOptions(): ParseSyncRulesOptions;
|
|
78
86
|
}
|
|
87
|
+
|
|
88
|
+
export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
|
|
@@ -3,7 +3,13 @@ import * as https from 'https';
|
|
|
3
3
|
import * as jose from 'jose';
|
|
4
4
|
import fetch from 'node-fetch';
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
ErrorCode,
|
|
8
|
+
LookupOptions,
|
|
9
|
+
makeHostnameLookupFunction,
|
|
10
|
+
ServiceAssertionError,
|
|
11
|
+
ServiceError
|
|
12
|
+
} from '@powersync/lib-services-framework';
|
|
7
13
|
import { KeyCollector, KeyResult } from './KeyCollector.js';
|
|
8
14
|
import { KeySpec } from './KeySpec.js';
|
|
9
15
|
|
|
@@ -24,14 +30,17 @@ export class RemoteJWKSCollector implements KeyCollector {
|
|
|
24
30
|
) {
|
|
25
31
|
try {
|
|
26
32
|
this.url = new URL(url);
|
|
27
|
-
} catch (e) {
|
|
28
|
-
throw new
|
|
33
|
+
} catch (e: any) {
|
|
34
|
+
throw new ServiceError(ErrorCode.PSYNC_S3102, `Invalid jwks_uri: ${JSON.stringify(url)} Details: ${e.message}`);
|
|
29
35
|
}
|
|
30
36
|
|
|
31
37
|
// We do support http here for self-hosting use cases.
|
|
32
38
|
// Management service restricts this to https for hosted versions.
|
|
33
39
|
if (this.url.protocol != 'https:' && this.url.protocol != 'http:') {
|
|
34
|
-
throw new
|
|
40
|
+
throw new ServiceError(
|
|
41
|
+
ErrorCode.PSYNC_S3103,
|
|
42
|
+
`Only http(s) is supported for jwks_uri, got: ${JSON.stringify(url)}`
|
|
43
|
+
);
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
this.agent = this.resolveAgent();
|
|
@@ -96,6 +105,9 @@ export class RemoteJWKSCollector implements KeyCollector {
|
|
|
96
105
|
|
|
97
106
|
/**
|
|
98
107
|
* Agent that uses a custom lookup function.
|
|
108
|
+
*
|
|
109
|
+
* This will synchronously raise an error if the URL contains an IP in the reject list.
|
|
110
|
+
* For domain names resolving to a rejected IP, that will fail when making the request.
|
|
99
111
|
*/
|
|
100
112
|
resolveAgent(): http.Agent | https.Agent {
|
|
101
113
|
const lookupOptions = this.options?.lookupOptions ?? { reject_ip_ranges: [] };
|
|
@@ -111,6 +123,7 @@ export class RemoteJWKSCollector implements KeyCollector {
|
|
|
111
123
|
case 'https:':
|
|
112
124
|
return new https.Agent(options);
|
|
113
125
|
}
|
|
114
|
-
|
|
126
|
+
// Already validated the URL before, so this is not expected
|
|
127
|
+
throw new ServiceAssertionError('http or or https is required for JWKS protocol');
|
|
115
128
|
}
|
|
116
129
|
}
|
package/src/entry/cli-entry.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { registerCompactAction } from './commands/compact-action.js';
|
|
|
6
6
|
import { registerMigrationAction } from './commands/migrate-action.js';
|
|
7
7
|
import { registerStartAction } from './commands/start-action.js';
|
|
8
8
|
import { registerTearDownAction } from './commands/teardown-action.js';
|
|
9
|
+
import { registerTestConnectionAction } from './commands/test-connection-action.js';
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* Generates a Commander program which serves as the entry point
|
|
@@ -20,6 +21,7 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
|
|
|
20
21
|
registerTearDownAction(entryProgram);
|
|
21
22
|
registerMigrationAction(entryProgram);
|
|
22
23
|
registerCompactAction(entryProgram);
|
|
24
|
+
registerTestConnectionAction(entryProgram);
|
|
23
25
|
|
|
24
26
|
if (startHandlers) {
|
|
25
27
|
registerStartAction(entryProgram, startHandlers);
|
|
@@ -2,6 +2,7 @@ import { Command } from 'commander';
|
|
|
2
2
|
|
|
3
3
|
import { teardown } from '../../runner/teardown.js';
|
|
4
4
|
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
5
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
5
6
|
|
|
6
7
|
const COMMAND_NAME = 'teardown';
|
|
7
8
|
|
|
@@ -15,7 +16,7 @@ export function registerTearDownAction(program: Command) {
|
|
|
15
16
|
.description('Terminate all replicating sync rules, clear remote configuration and remove all data')
|
|
16
17
|
.action(async (ack, options) => {
|
|
17
18
|
if (ack !== 'TEARDOWN') {
|
|
18
|
-
throw new
|
|
19
|
+
throw new ServiceError(ErrorCode.PSYNC_S0102, 'TEARDOWN was not acknowledged.');
|
|
19
20
|
}
|
|
20
21
|
|
|
21
22
|
await teardown(extractRunnerOptions(options));
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
|
|
3
|
+
import { container, logger } from '@powersync/lib-services-framework';
|
|
4
|
+
import * as system from '../../system/system-index.js';
|
|
5
|
+
import * as utils from '../../util/util-index.js';
|
|
6
|
+
|
|
7
|
+
import { modules, ReplicationEngine } from '../../index.js';
|
|
8
|
+
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
9
|
+
|
|
10
|
+
const COMMAND_NAME = 'test-connection';
|
|
11
|
+
|
|
12
|
+
export function registerTestConnectionAction(program: Command) {
|
|
13
|
+
const testConnectionCommand = program.command(COMMAND_NAME);
|
|
14
|
+
|
|
15
|
+
wrapConfigCommand(testConnectionCommand);
|
|
16
|
+
|
|
17
|
+
return testConnectionCommand.description('Test connection').action(async (options) => {
|
|
18
|
+
try {
|
|
19
|
+
const config = await utils.loadConfig(extractRunnerOptions(options));
|
|
20
|
+
const serviceContext = new system.ServiceContextContainer(config);
|
|
21
|
+
|
|
22
|
+
const replication = new ReplicationEngine();
|
|
23
|
+
serviceContext.register(ReplicationEngine, replication);
|
|
24
|
+
|
|
25
|
+
// Register modules in order to load the correct config
|
|
26
|
+
const moduleManager = container.getImplementation(modules.ModuleManager);
|
|
27
|
+
await moduleManager.initialize(serviceContext);
|
|
28
|
+
|
|
29
|
+
// Start the storage engine in order to create the appropriate BucketStorage
|
|
30
|
+
await serviceContext.lifeCycleEngine.start();
|
|
31
|
+
|
|
32
|
+
logger.info('Testing connection...');
|
|
33
|
+
const results = await replication.testConnection();
|
|
34
|
+
logger.info(`Connection succeeded to ${results.map((r) => r.connectionDescription).join(', ')}`);
|
|
35
|
+
process.exit(0);
|
|
36
|
+
} catch (e) {
|
|
37
|
+
logger.error(`Connection failed: ${e.message}`);
|
|
38
|
+
process.exit(1);
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
}
|
package/src/metrics/Metrics.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
|
3
3
|
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
|
4
4
|
import { Resource } from '@opentelemetry/resources';
|
|
5
5
|
import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
|
|
6
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
6
|
+
import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
7
7
|
import * as storage from '../storage/storage-index.js';
|
|
8
8
|
import * as util from '../util/util-index.js';
|
|
9
9
|
|
|
@@ -132,7 +132,7 @@ export class Metrics {
|
|
|
132
132
|
|
|
133
133
|
public static getInstance(): Metrics {
|
|
134
134
|
if (!Metrics.instance) {
|
|
135
|
-
throw new
|
|
135
|
+
throw new ServiceAssertionError('Metrics have not been initialized');
|
|
136
136
|
}
|
|
137
137
|
|
|
138
138
|
return Metrics.instance;
|
|
@@ -6,6 +6,7 @@ import { StorageEngine } from '../storage/storage-index.js';
|
|
|
6
6
|
import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
|
|
7
7
|
import { AbstractReplicationJob } from './AbstractReplicationJob.js';
|
|
8
8
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
9
|
+
import { ConnectionTestResult } from './ReplicationModule.js';
|
|
9
10
|
|
|
10
11
|
// 5 minutes
|
|
11
12
|
const PING_INTERVAL = 1_000_000_000n * 300n;
|
|
@@ -92,21 +93,27 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
92
93
|
|
|
93
94
|
private async runLoop() {
|
|
94
95
|
const syncRules = await this.syncRuleProvider.get();
|
|
96
|
+
|
|
95
97
|
let configuredLock: storage.ReplicationLock | undefined = undefined;
|
|
96
98
|
if (syncRules != null) {
|
|
97
99
|
this.logger.info('Loaded sync rules');
|
|
98
100
|
try {
|
|
99
101
|
// Configure new sync rules, if they have changed.
|
|
100
102
|
// In that case, also immediately take out a lock, so that another process doesn't start replication on it.
|
|
101
|
-
|
|
102
|
-
|
|
103
|
+
|
|
104
|
+
const { lock } = await this.storage.configureSyncRules({
|
|
105
|
+
content: syncRules,
|
|
106
|
+
lock: true,
|
|
107
|
+
validate: this.syncRuleProvider.exitOnError
|
|
103
108
|
});
|
|
104
109
|
if (lock) {
|
|
105
110
|
configuredLock = lock;
|
|
106
111
|
}
|
|
107
112
|
} catch (e) {
|
|
108
|
-
// Log
|
|
113
|
+
// Log and re-raise to exit.
|
|
114
|
+
// Should only reach this due to validation errors if exit_on_error is true.
|
|
109
115
|
this.logger.error(`Failed to update sync rules from configuration`, e);
|
|
116
|
+
throw e;
|
|
110
117
|
}
|
|
111
118
|
} else {
|
|
112
119
|
this.logger.info('No sync rules configured - configure via API');
|
|
@@ -225,4 +232,6 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
225
232
|
this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
|
|
226
233
|
}
|
|
227
234
|
}
|
|
235
|
+
|
|
236
|
+
abstract testConnection(): Promise<ConnectionTestResult>;
|
|
228
237
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { AbstractReplicator } from './AbstractReplicator.js';
|
|
3
|
+
import { ConnectionTestResult } from './ReplicationModule.js';
|
|
3
4
|
|
|
4
5
|
export class ReplicationEngine {
|
|
5
6
|
private readonly replicators: Map<string, AbstractReplicator> = new Map();
|
|
@@ -40,4 +41,8 @@ export class ReplicationEngine {
|
|
|
40
41
|
}
|
|
41
42
|
logger.info('Successfully shut down Replication Engine.');
|
|
42
43
|
}
|
|
44
|
+
|
|
45
|
+
public async testConnection(): Promise<ConnectionTestResult[]> {
|
|
46
|
+
return await Promise.all([...this.replicators.values()].map((replicator) => replicator.testConnection()));
|
|
47
|
+
}
|
|
43
48
|
}
|
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
|
|
2
2
|
import * as t from 'ts-codec';
|
|
3
3
|
|
|
4
|
+
import { schema } from '@powersync/lib-services-framework';
|
|
4
5
|
import * as types from '@powersync/service-types';
|
|
5
6
|
import * as api from '../api/api-index.js';
|
|
6
7
|
import * as modules from '../modules/modules-index.js';
|
|
7
8
|
import * as system from '../system/system-index.js';
|
|
8
|
-
import { schema } from '@powersync/lib-services-framework';
|
|
9
9
|
import { AbstractReplicator } from './AbstractReplicator.js';
|
|
10
|
-
|
|
10
|
+
|
|
11
|
+
export interface ConnectionTestResult {
|
|
12
|
+
/**
|
|
13
|
+
* Connection URI or hostname.
|
|
14
|
+
*/
|
|
15
|
+
connectionDescription: string;
|
|
16
|
+
}
|
|
11
17
|
|
|
12
18
|
/**
|
|
13
19
|
* Provides a common interface for testing the connection to a DataSource.
|
|
@@ -17,7 +23,7 @@ export interface ConnectionTester<TConfig extends DataSourceConfig> {
|
|
|
17
23
|
* Confirm if a connection can be established to the datasource for the provided datasource configuration
|
|
18
24
|
* @param config
|
|
19
25
|
*/
|
|
20
|
-
testConnection(config: TConfig): Promise<
|
|
26
|
+
testConnection(config: TConfig): Promise<ConnectionTestResult>;
|
|
21
27
|
}
|
|
22
28
|
|
|
23
29
|
export interface ReplicationModuleOptions extends modules.AbstractModuleOptions {
|
|
@@ -58,7 +64,7 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig>
|
|
|
58
64
|
*/
|
|
59
65
|
protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
|
|
60
66
|
|
|
61
|
-
public abstract testConnection(config: TConfig): Promise<
|
|
67
|
+
public abstract testConnection(config: TConfig): Promise<ConnectionTestResult>;
|
|
62
68
|
|
|
63
69
|
/**
|
|
64
70
|
* Register this module's Replicators and RouteAPI adapters if the required configuration is present.
|
|
@@ -81,16 +87,12 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig>
|
|
|
81
87
|
);
|
|
82
88
|
}
|
|
83
89
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
this.decodeConfig(baseMatchingConfig);
|
|
90
|
+
const baseMatchingConfig = matchingConfig[0] as TConfig;
|
|
91
|
+
// If decoding fails, this will raise a hard error, and stop the service.
|
|
92
|
+
this.decodeConfig(baseMatchingConfig);
|
|
88
93
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
} catch (e) {
|
|
92
|
-
this.logger.error('Failed to initialize.', e);
|
|
93
|
-
}
|
|
94
|
+
context.replicationEngine?.register(this.createReplicator(context));
|
|
95
|
+
context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
|
|
94
96
|
}
|
|
95
97
|
|
|
96
98
|
protected decodeConfig(config: TConfig): void {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { errors, router, schema } from '@powersync/lib-services-framework';
|
|
1
|
+
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
|
|
3
3
|
import { internal_routes } from '@powersync/service-types';
|
|
4
4
|
|
|
@@ -120,15 +120,18 @@ export const reprocess = routeDefinition({
|
|
|
120
120
|
|
|
121
121
|
const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
|
|
122
122
|
if (active == null) {
|
|
123
|
-
throw new errors.
|
|
123
|
+
throw new errors.ServiceError({
|
|
124
124
|
status: 422,
|
|
125
|
-
code:
|
|
125
|
+
code: ErrorCode.PSYNC_S4104,
|
|
126
126
|
description: 'No active sync rules'
|
|
127
127
|
});
|
|
128
128
|
}
|
|
129
129
|
|
|
130
130
|
const new_rules = await activeBucketStorage.updateSyncRules({
|
|
131
|
-
content: active.sync_rules.content
|
|
131
|
+
content: active.sync_rules.content,
|
|
132
|
+
// These sync rules already passed validation. But if the rules are not valid anymore due
|
|
133
|
+
// to a service change, we do want to report the error here.
|
|
134
|
+
validate: true
|
|
132
135
|
});
|
|
133
136
|
|
|
134
137
|
const baseConfig = await apiHandler.getSourceConfig();
|
|
@@ -25,7 +25,7 @@ export const writeCheckpoint = routeDefinition({
|
|
|
25
25
|
// Since we don't use LSNs anymore, the only way to get that is to wait.
|
|
26
26
|
const start = Date.now();
|
|
27
27
|
|
|
28
|
-
const head = await apiHandler.
|
|
28
|
+
const head = await apiHandler.createReplicationHead(async (head) => head);
|
|
29
29
|
|
|
30
30
|
const timeout = 50_000;
|
|
31
31
|
|
|
@@ -56,25 +56,14 @@ export const writeCheckpoint2 = routeDefinition({
|
|
|
56
56
|
|
|
57
57
|
const apiHandler = service_context.routerEngine!.getAPI();
|
|
58
58
|
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
storageEngine: { activeBucketStorage }
|
|
65
|
-
} = service_context;
|
|
66
|
-
|
|
67
|
-
const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
|
|
68
|
-
if (!activeSyncRules) {
|
|
69
|
-
throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
|
|
73
|
-
const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
|
|
74
|
-
user_id: full_user_id,
|
|
75
|
-
heads: { '1': currentCheckpoint }
|
|
59
|
+
const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
|
|
60
|
+
userId: user_id,
|
|
61
|
+
clientId: payload.params.client_id,
|
|
62
|
+
api: apiHandler,
|
|
63
|
+
storage: service_context.storageEngine.activeBucketStorage
|
|
76
64
|
});
|
|
77
|
-
|
|
65
|
+
|
|
66
|
+
logger.info(`Write checkpoint for ${user_id}/${payload.params.client_id}: ${writeCheckpoint} | ${replicationHead}`);
|
|
78
67
|
|
|
79
68
|
return {
|
|
80
69
|
write_checkpoint: String(writeCheckpoint)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { errors, logger, schema } from '@powersync/lib-services-framework';
|
|
1
|
+
import { ErrorCode, errors, logger, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
3
|
import { serialize } from 'bson';
|
|
4
4
|
|
|
@@ -34,9 +34,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
34
34
|
|
|
35
35
|
if (routerEngine!.closed) {
|
|
36
36
|
responder.onError(
|
|
37
|
-
new errors.
|
|
37
|
+
new errors.ServiceError({
|
|
38
38
|
status: 503,
|
|
39
|
-
code:
|
|
39
|
+
code: ErrorCode.PSYNC_S2003,
|
|
40
40
|
description: 'Service temporarily unavailable'
|
|
41
41
|
})
|
|
42
42
|
);
|
|
@@ -53,9 +53,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
53
53
|
const cp = await activeBucketStorage.getActiveCheckpoint();
|
|
54
54
|
if (!cp.hasSyncRules()) {
|
|
55
55
|
responder.onError(
|
|
56
|
-
new errors.
|
|
56
|
+
new errors.ServiceError({
|
|
57
57
|
status: 500,
|
|
58
|
-
code:
|
|
58
|
+
code: ErrorCode.PSYNC_S2302,
|
|
59
59
|
description: 'No sync rules available'
|
|
60
60
|
})
|
|
61
61
|
);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { errors, router, schema } from '@powersync/lib-services-framework';
|
|
1
|
+
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
3
3
|
import type { FastifyPluginAsync } from 'fastify';
|
|
4
4
|
import * as t from 'ts-codec';
|
|
@@ -43,9 +43,9 @@ export const deploySyncRules = routeDefinition({
|
|
|
43
43
|
|
|
44
44
|
if (service_context.configuration.sync_rules.present) {
|
|
45
45
|
// If sync rules are configured via the config, disable deploy via the API.
|
|
46
|
-
throw new errors.
|
|
46
|
+
throw new errors.ServiceError({
|
|
47
47
|
status: 422,
|
|
48
|
-
code:
|
|
48
|
+
code: ErrorCode.PSYNC_S4105,
|
|
49
49
|
description: 'Sync rules API disabled',
|
|
50
50
|
details: 'Use the management API to deploy sync rules'
|
|
51
51
|
});
|
|
@@ -60,16 +60,18 @@ export const deploySyncRules = routeDefinition({
|
|
|
60
60
|
schema: undefined
|
|
61
61
|
});
|
|
62
62
|
} catch (e) {
|
|
63
|
-
throw new errors.
|
|
63
|
+
throw new errors.ServiceError({
|
|
64
64
|
status: 422,
|
|
65
|
-
code:
|
|
65
|
+
code: ErrorCode.PSYNC_R0001,
|
|
66
66
|
description: 'Sync rules parsing failed',
|
|
67
67
|
details: e.message
|
|
68
68
|
});
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
|
|
72
|
-
content: content
|
|
72
|
+
content: content,
|
|
73
|
+
// Aready validated above
|
|
74
|
+
validate: false
|
|
73
75
|
});
|
|
74
76
|
|
|
75
77
|
return {
|
|
@@ -112,9 +114,9 @@ export const currentSyncRules = routeDefinition({
|
|
|
112
114
|
|
|
113
115
|
const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
|
|
114
116
|
if (!sync_rules) {
|
|
115
|
-
throw new errors.
|
|
117
|
+
throw new errors.ServiceError({
|
|
116
118
|
status: 422,
|
|
117
|
-
code:
|
|
119
|
+
code: ErrorCode.PSYNC_S4104,
|
|
118
120
|
description: 'No active sync rules'
|
|
119
121
|
});
|
|
120
122
|
}
|
|
@@ -159,15 +161,18 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
159
161
|
const apiHandler = payload.context.service_context.routerEngine!.getAPI();
|
|
160
162
|
const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
|
|
161
163
|
if (sync_rules == null) {
|
|
162
|
-
throw new errors.
|
|
164
|
+
throw new errors.ServiceError({
|
|
163
165
|
status: 422,
|
|
164
|
-
code:
|
|
166
|
+
code: ErrorCode.PSYNC_S4104,
|
|
165
167
|
description: 'No active sync rules'
|
|
166
168
|
});
|
|
167
169
|
}
|
|
168
170
|
|
|
169
171
|
const new_rules = await activeBucketStorage.updateSyncRules({
|
|
170
|
-
content: sync_rules.sync_rules.content
|
|
172
|
+
content: sync_rules.sync_rules.content,
|
|
173
|
+
// These sync rules already passed validation. But if the rules are not valid anymore due
|
|
174
|
+
// to a service change, we do want to report the error here.
|
|
175
|
+
validate: true
|
|
171
176
|
});
|
|
172
177
|
return {
|
|
173
178
|
slot_name: new_rules.slot_name
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { errors, logger, router, schema } from '@powersync/lib-services-framework';
|
|
1
|
+
import { ErrorCode, errors, logger, router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
3
|
import { Readable } from 'stream';
|
|
4
4
|
|
|
@@ -26,9 +26,9 @@ export const syncStreamed = routeDefinition({
|
|
|
26
26
|
const clientId = payload.params.client_id;
|
|
27
27
|
|
|
28
28
|
if (routerEngine!.closed) {
|
|
29
|
-
throw new errors.
|
|
29
|
+
throw new errors.ServiceError({
|
|
30
30
|
status: 503,
|
|
31
|
-
code:
|
|
31
|
+
code: ErrorCode.PSYNC_S2003,
|
|
32
32
|
description: 'Service temporarily unavailable'
|
|
33
33
|
});
|
|
34
34
|
}
|
|
@@ -39,9 +39,9 @@ export const syncStreamed = routeDefinition({
|
|
|
39
39
|
// Sanity check before we start the stream
|
|
40
40
|
const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
|
|
41
41
|
if (!cp.hasSyncRules()) {
|
|
42
|
-
throw new errors.
|
|
42
|
+
throw new errors.ServiceError({
|
|
43
43
|
status: 500,
|
|
44
|
-
code:
|
|
44
|
+
code: ErrorCode.PSYNC_S2302,
|
|
45
45
|
description: 'No sync rules available'
|
|
46
46
|
});
|
|
47
47
|
}
|
|
@@ -62,16 +62,16 @@ export function registerFastifyRoutes(
|
|
|
62
62
|
});
|
|
63
63
|
}
|
|
64
64
|
} catch (ex) {
|
|
65
|
-
const
|
|
66
|
-
logger.error(`Request failed`,
|
|
65
|
+
const serviceError = errors.asServiceError(ex);
|
|
66
|
+
logger.error(`Request failed`, serviceError);
|
|
67
67
|
|
|
68
68
|
response = new router.RouterResponse({
|
|
69
|
-
status:
|
|
69
|
+
status: serviceError.errorData.status || 500,
|
|
70
70
|
headers: {
|
|
71
71
|
'Content-Type': 'application/json'
|
|
72
72
|
},
|
|
73
73
|
data: {
|
|
74
|
-
error:
|
|
74
|
+
error: serviceError.errorData
|
|
75
75
|
}
|
|
76
76
|
});
|
|
77
77
|
}
|
|
@@ -60,13 +60,26 @@ export interface BucketStorageFactoryListener extends DisposableListener {
|
|
|
60
60
|
replicationEvent: (event: ReplicationEventPayload) => void;
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
export interface BucketStorageSystemIdentifier {
|
|
64
|
+
/**
|
|
65
|
+
* A unique identifier for the system used for storage.
|
|
66
|
+
* For Postgres this can be the cluster `system_identifier` and database name.
|
|
67
|
+
* For MongoDB this can be the replica set name.
|
|
68
|
+
*/
|
|
69
|
+
id: string;
|
|
70
|
+
/**
|
|
71
|
+
* A unique type for the storage implementation.
|
|
72
|
+
* e.g. `mongodb`, `postgresql`.
|
|
73
|
+
*/
|
|
74
|
+
type: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
63
77
|
export interface BucketStorageFactory extends AsyncDisposableObserverClient<BucketStorageFactoryListener> {
|
|
64
78
|
/**
|
|
65
79
|
* Update sync rules from configuration, if changed.
|
|
66
80
|
*/
|
|
67
81
|
configureSyncRules(
|
|
68
|
-
|
|
69
|
-
options?: { lock?: boolean }
|
|
82
|
+
options: UpdateSyncRulesOptions
|
|
70
83
|
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>;
|
|
71
84
|
|
|
72
85
|
/**
|
|
@@ -76,6 +89,8 @@ export interface BucketStorageFactory extends AsyncDisposableObserverClient<Buck
|
|
|
76
89
|
|
|
77
90
|
/**
|
|
78
91
|
* Deploy new sync rules.
|
|
92
|
+
*
|
|
93
|
+
* Similar to configureSyncRules, but applies the update unconditionally.
|
|
79
94
|
*/
|
|
80
95
|
updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
81
96
|
|
|
@@ -143,6 +158,11 @@ export interface BucketStorageFactory extends AsyncDisposableObserverClient<Buck
|
|
|
143
158
|
* Get the unique identifier for this instance of Powersync
|
|
144
159
|
*/
|
|
145
160
|
getPowerSyncInstanceId(): Promise<string>;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Get a unique identifier for the system used for storage.
|
|
164
|
+
*/
|
|
165
|
+
getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
146
166
|
}
|
|
147
167
|
|
|
148
168
|
export interface ReplicationCheckpoint {
|
|
@@ -213,6 +233,7 @@ export interface PersistedSyncRules {
|
|
|
213
233
|
export interface UpdateSyncRulesOptions {
|
|
214
234
|
content: string;
|
|
215
235
|
lock?: boolean;
|
|
236
|
+
validate?: boolean;
|
|
216
237
|
}
|
|
217
238
|
|
|
218
239
|
export interface SyncRulesBucketStorageOptions {
|
|
@@ -367,6 +388,20 @@ export interface BucketBatchStorageListener extends DisposableListener {
|
|
|
367
388
|
replicationEvent: (payload: ReplicationEventPayload) => void;
|
|
368
389
|
}
|
|
369
390
|
|
|
391
|
+
export interface BucketBatchCommitOptions {
|
|
392
|
+
/**
|
|
393
|
+
* Creates a new checkpoint even if there were no persisted operations.
|
|
394
|
+
* Defaults to true.
|
|
395
|
+
*/
|
|
396
|
+
createEmptyCheckpoints?: boolean;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
export type ResolvedBucketBatchCommitOptions = Required<BucketBatchCommitOptions>;
|
|
400
|
+
|
|
401
|
+
export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
|
|
402
|
+
createEmptyCheckpoints: true
|
|
403
|
+
};
|
|
404
|
+
|
|
370
405
|
export interface BucketStorageBatch extends DisposableObserverClient<BucketBatchStorageListener> {
|
|
371
406
|
/**
|
|
372
407
|
* Save an op, and potentially flush.
|
|
@@ -398,11 +433,11 @@ export interface BucketStorageBatch extends DisposableObserverClient<BucketBatch
|
|
|
398
433
|
flush(): Promise<FlushedResult | null>;
|
|
399
434
|
|
|
400
435
|
/**
|
|
401
|
-
* Flush and commit any saved ops. This creates a new checkpoint.
|
|
436
|
+
* Flush and commit any saved ops. This creates a new checkpoint by default.
|
|
402
437
|
*
|
|
403
438
|
* Only call this after a transaction.
|
|
404
439
|
*/
|
|
405
|
-
commit(lsn: string): Promise<boolean>;
|
|
440
|
+
commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
|
|
406
441
|
|
|
407
442
|
/**
|
|
408
443
|
* Advance the checkpoint LSN position, without any associated op.
|