@powersync/service-core 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/CHANGELOG.md +48 -8
  2. package/dist/api/RouteAPI.d.ts +8 -0
  3. package/dist/auth/CachedKeyCollector.js +26 -25
  4. package/dist/auth/CachedKeyCollector.js.map +1 -1
  5. package/dist/auth/CompoundKeyCollector.js +1 -0
  6. package/dist/auth/CompoundKeyCollector.js.map +1 -1
  7. package/dist/auth/KeySpec.js +3 -0
  8. package/dist/auth/KeySpec.js.map +1 -1
  9. package/dist/auth/KeyStore.js +4 -0
  10. package/dist/auth/KeyStore.js.map +1 -1
  11. package/dist/auth/LeakyBucket.js +5 -0
  12. package/dist/auth/LeakyBucket.js.map +1 -1
  13. package/dist/auth/RemoteJWKSCollector.d.ts +3 -0
  14. package/dist/auth/RemoteJWKSCollector.js +12 -5
  15. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  16. package/dist/auth/StaticKeyCollector.js +1 -0
  17. package/dist/auth/StaticKeyCollector.js.map +1 -1
  18. package/dist/auth/StaticSupabaseKeyCollector.js +1 -0
  19. package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
  20. package/dist/entry/cli-entry.js +2 -0
  21. package/dist/entry/cli-entry.js.map +1 -1
  22. package/dist/entry/commands/teardown-action.js +2 -1
  23. package/dist/entry/commands/teardown-action.js.map +1 -1
  24. package/dist/entry/commands/test-connection-action.d.ts +2 -0
  25. package/dist/entry/commands/test-connection-action.js +32 -0
  26. package/dist/entry/commands/test-connection-action.js.map +1 -0
  27. package/dist/metrics/Metrics.js +37 -3
  28. package/dist/metrics/Metrics.js.map +1 -1
  29. package/dist/modules/AbstractModule.js +2 -0
  30. package/dist/modules/AbstractModule.js.map +1 -1
  31. package/dist/modules/ModuleManager.js +1 -3
  32. package/dist/modules/ModuleManager.js.map +1 -1
  33. package/dist/replication/AbstractReplicationJob.js +4 -2
  34. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  35. package/dist/replication/AbstractReplicator.d.ts +2 -0
  36. package/dist/replication/AbstractReplicator.js +18 -12
  37. package/dist/replication/AbstractReplicator.js.map +1 -1
  38. package/dist/replication/ReplicationEngine.d.ts +2 -0
  39. package/dist/replication/ReplicationEngine.js +4 -3
  40. package/dist/replication/ReplicationEngine.js.map +1 -1
  41. package/dist/replication/ReplicationModule.d.ts +8 -2
  42. package/dist/replication/ReplicationModule.js +9 -11
  43. package/dist/replication/ReplicationModule.js.map +1 -1
  44. package/dist/routes/RouterEngine.js +8 -0
  45. package/dist/routes/RouterEngine.js.map +1 -1
  46. package/dist/routes/configure-fastify.d.ts +3 -3
  47. package/dist/routes/endpoints/admin.d.ts +6 -6
  48. package/dist/routes/endpoints/admin.js +7 -4
  49. package/dist/routes/endpoints/admin.js.map +1 -1
  50. package/dist/routes/endpoints/checkpointing.js +14 -84
  51. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  52. package/dist/routes/endpoints/socket-route.js +5 -5
  53. package/dist/routes/endpoints/socket-route.js.map +1 -1
  54. package/dist/routes/endpoints/sync-rules.js +16 -11
  55. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  56. package/dist/routes/endpoints/sync-stream.js +5 -5
  57. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  58. package/dist/routes/route-register.js +4 -4
  59. package/dist/routes/route-register.js.map +1 -1
  60. package/dist/runner/teardown.js +2 -2
  61. package/dist/runner/teardown.js.map +1 -1
  62. package/dist/storage/BucketStorage.d.ts +32 -5
  63. package/dist/storage/BucketStorage.js +3 -0
  64. package/dist/storage/BucketStorage.js.map +1 -1
  65. package/dist/storage/ChecksumCache.js +12 -7
  66. package/dist/storage/ChecksumCache.js.map +1 -1
  67. package/dist/storage/SourceTable.js +32 -25
  68. package/dist/storage/SourceTable.js.map +1 -1
  69. package/dist/storage/StorageEngine.js +4 -3
  70. package/dist/storage/StorageEngine.js.map +1 -1
  71. package/dist/storage/bson.d.ts +5 -3
  72. package/dist/storage/bson.js.map +1 -1
  73. package/dist/sync/BroadcastIterable.js +4 -3
  74. package/dist/sync/BroadcastIterable.js.map +1 -1
  75. package/dist/sync/LastValueSink.js +2 -0
  76. package/dist/sync/LastValueSink.js.map +1 -1
  77. package/dist/sync/RequestTracker.js +2 -4
  78. package/dist/sync/RequestTracker.js.map +1 -1
  79. package/dist/sync/merge.js +4 -0
  80. package/dist/sync/merge.js.map +1 -1
  81. package/dist/sync/sync.js +2 -2
  82. package/dist/sync/sync.js.map +1 -1
  83. package/dist/sync/util.js +2 -2
  84. package/dist/sync/util.js.map +1 -1
  85. package/dist/system/ServiceContext.js +3 -0
  86. package/dist/system/ServiceContext.js.map +1 -1
  87. package/dist/util/Mutex.js +5 -0
  88. package/dist/util/Mutex.js.map +1 -1
  89. package/dist/util/checkpointing.d.ts +13 -0
  90. package/dist/util/checkpointing.js +92 -0
  91. package/dist/util/checkpointing.js.map +1 -0
  92. package/dist/util/config/compound-config-collector.js +3 -1
  93. package/dist/util/config/compound-config-collector.js.map +1 -1
  94. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  95. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  96. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  97. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  98. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +1 -0
  99. package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
  100. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  101. package/dist/util/config/sync-rules/sync-rules-provider.js +4 -0
  102. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  103. package/dist/util/config/types.d.ts +1 -0
  104. package/dist/util/memory-tracking.js +1 -1
  105. package/dist/util/memory-tracking.js.map +1 -1
  106. package/dist/util/util-index.d.ts +1 -0
  107. package/dist/util/util-index.js +1 -0
  108. package/dist/util/util-index.js.map +1 -1
  109. package/dist/util/utils.d.ts +0 -1
  110. package/dist/util/utils.js +2 -10
  111. package/dist/util/utils.js.map +1 -1
  112. package/package.json +5 -5
  113. package/src/api/RouteAPI.ts +10 -0
  114. package/src/auth/RemoteJWKSCollector.ts +18 -5
  115. package/src/entry/cli-entry.ts +2 -0
  116. package/src/entry/commands/teardown-action.ts +2 -1
  117. package/src/entry/commands/test-connection-action.ts +41 -0
  118. package/src/metrics/Metrics.ts +2 -2
  119. package/src/replication/AbstractReplicator.ts +12 -3
  120. package/src/replication/ReplicationEngine.ts +5 -0
  121. package/src/replication/ReplicationModule.ts +15 -13
  122. package/src/routes/endpoints/admin.ts +7 -4
  123. package/src/routes/endpoints/checkpointing.ts +8 -19
  124. package/src/routes/endpoints/socket-route.ts +5 -5
  125. package/src/routes/endpoints/sync-rules.ts +16 -11
  126. package/src/routes/endpoints/sync-stream.ts +5 -5
  127. package/src/routes/route-register.ts +4 -4
  128. package/src/storage/BucketStorage.ts +39 -4
  129. package/src/storage/bson.ts +9 -7
  130. package/src/util/checkpointing.ts +43 -0
  131. package/src/util/config/compound-config-collector.ts +2 -1
  132. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  133. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  134. package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +1 -0
  135. package/src/util/config/sync-rules/sync-rules-provider.ts +6 -0
  136. package/src/util/config/types.ts +1 -0
  137. package/src/util/memory-tracking.ts +2 -2
  138. package/src/util/util-index.ts +1 -0
  139. package/src/util/utils.ts +2 -11
  140. package/tsconfig.tsbuildinfo +1 -1
package/package.json CHANGED
@@ -5,7 +5,7 @@
5
5
  "publishConfig": {
6
6
  "access": "public"
7
7
  },
8
- "version": "0.0.0-dev-20250117095455",
8
+ "version": "0.0.0-dev-20250214100224",
9
9
  "main": "dist/index.js",
10
10
  "license": "FSL-1.1-Apache-2.0",
11
11
  "type": "module",
@@ -32,11 +32,11 @@
32
32
  "uuid": "^9.0.1",
33
33
  "winston": "^3.13.0",
34
34
  "yaml": "^2.3.2",
35
- "@powersync/lib-services-framework": "0.0.0-dev-20250117095455",
35
+ "@powersync/lib-services-framework": "0.5.1",
36
36
  "@powersync/service-jsonbig": "0.17.10",
37
- "@powersync/service-rsocket-router": "0.0.0-dev-20250117095455",
38
- "@powersync/service-sync-rules": "0.23.1",
39
- "@powersync/service-types": "0.0.0-dev-20250117095455"
37
+ "@powersync/service-rsocket-router": "0.0.18",
38
+ "@powersync/service-sync-rules": "0.23.4",
39
+ "@powersync/service-types": "0.8.0"
40
40
  },
41
41
  "devDependencies": {
42
42
  "@types/async": "^3.2.24",
@@ -54,6 +54,14 @@ export interface RouteAPI {
54
54
  */
55
55
  getReplicationHead(): Promise<string>;
56
56
 
57
+ /**
58
+ * Get the current LSN or equivalent replication HEAD position identifier.
59
+ *
60
+ * The position is provided to the callback. After the callback returns,
61
+ * the replication head or a greater one will be streamed on the replication stream.
62
+ */
63
+ createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
64
+
57
65
  /**
58
66
  * @returns The schema for tables inside the connected database. This is typically
59
67
  * used to validate sync rules.
@@ -76,3 +84,5 @@ export interface RouteAPI {
76
84
  */
77
85
  getParseSyncRulesOptions(): ParseSyncRulesOptions;
78
86
  }
87
+
88
+ export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
@@ -3,7 +3,13 @@ import * as https from 'https';
3
3
  import * as jose from 'jose';
4
4
  import fetch from 'node-fetch';
5
5
 
6
- import { LookupOptions, makeHostnameLookupFunction } from '@powersync/lib-services-framework';
6
+ import {
7
+ ErrorCode,
8
+ LookupOptions,
9
+ makeHostnameLookupFunction,
10
+ ServiceAssertionError,
11
+ ServiceError
12
+ } from '@powersync/lib-services-framework';
7
13
  import { KeyCollector, KeyResult } from './KeyCollector.js';
8
14
  import { KeySpec } from './KeySpec.js';
9
15
 
@@ -24,14 +30,17 @@ export class RemoteJWKSCollector implements KeyCollector {
24
30
  ) {
25
31
  try {
26
32
  this.url = new URL(url);
27
- } catch (e) {
28
- throw new Error(`Invalid jwks_uri: ${url}`);
33
+ } catch (e: any) {
34
+ throw new ServiceError(ErrorCode.PSYNC_S3102, `Invalid jwks_uri: ${JSON.stringify(url)} Details: ${e.message}`);
29
35
  }
30
36
 
31
37
  // We do support http here for self-hosting use cases.
32
38
  // Management service restricts this to https for hosted versions.
33
39
  if (this.url.protocol != 'https:' && this.url.protocol != 'http:') {
34
- throw new Error(`Only http(s) is supported for jwks_uri, got: ${url}`);
40
+ throw new ServiceError(
41
+ ErrorCode.PSYNC_S3103,
42
+ `Only http(s) is supported for jwks_uri, got: ${JSON.stringify(url)}`
43
+ );
35
44
  }
36
45
 
37
46
  this.agent = this.resolveAgent();
@@ -96,6 +105,9 @@ export class RemoteJWKSCollector implements KeyCollector {
96
105
 
97
106
  /**
98
107
  * Agent that uses a custom lookup function.
108
+ *
109
+ * This will synchronously raise an error if the URL contains an IP in the reject list.
110
+ * For domain names resolving to a rejected IP, that will fail when making the request.
99
111
  */
100
112
  resolveAgent(): http.Agent | https.Agent {
101
113
  const lookupOptions = this.options?.lookupOptions ?? { reject_ip_ranges: [] };
@@ -111,6 +123,7 @@ export class RemoteJWKSCollector implements KeyCollector {
111
123
  case 'https:':
112
124
  return new https.Agent(options);
113
125
  }
114
- throw new Error('http or or https is required for protocol');
126
+ // Already validated the URL before, so this is not expected
127
+ throw new ServiceAssertionError('http or or https is required for JWKS protocol');
115
128
  }
116
129
  }
@@ -6,6 +6,7 @@ import { registerCompactAction } from './commands/compact-action.js';
6
6
  import { registerMigrationAction } from './commands/migrate-action.js';
7
7
  import { registerStartAction } from './commands/start-action.js';
8
8
  import { registerTearDownAction } from './commands/teardown-action.js';
9
+ import { registerTestConnectionAction } from './commands/test-connection-action.js';
9
10
 
10
11
  /**
11
12
  * Generates a Commander program which serves as the entry point
@@ -20,6 +21,7 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
20
21
  registerTearDownAction(entryProgram);
21
22
  registerMigrationAction(entryProgram);
22
23
  registerCompactAction(entryProgram);
24
+ registerTestConnectionAction(entryProgram);
23
25
 
24
26
  if (startHandlers) {
25
27
  registerStartAction(entryProgram, startHandlers);
@@ -2,6 +2,7 @@ import { Command } from 'commander';
2
2
 
3
3
  import { teardown } from '../../runner/teardown.js';
4
4
  import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
5
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
5
6
 
6
7
  const COMMAND_NAME = 'teardown';
7
8
 
@@ -15,7 +16,7 @@ export function registerTearDownAction(program: Command) {
15
16
  .description('Terminate all replicating sync rules, clear remote configuration and remove all data')
16
17
  .action(async (ack, options) => {
17
18
  if (ack !== 'TEARDOWN') {
18
- throw new Error('TEARDOWN was not acknowledged.');
19
+ throw new ServiceError(ErrorCode.PSYNC_S0102, 'TEARDOWN was not acknowledged.');
19
20
  }
20
21
 
21
22
  await teardown(extractRunnerOptions(options));
@@ -0,0 +1,41 @@
1
+ import { Command } from 'commander';
2
+
3
+ import { container, logger } from '@powersync/lib-services-framework';
4
+ import * as system from '../../system/system-index.js';
5
+ import * as utils from '../../util/util-index.js';
6
+
7
+ import { modules, ReplicationEngine } from '../../index.js';
8
+ import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
9
+
10
+ const COMMAND_NAME = 'test-connection';
11
+
12
+ export function registerTestConnectionAction(program: Command) {
13
+ const testConnectionCommand = program.command(COMMAND_NAME);
14
+
15
+ wrapConfigCommand(testConnectionCommand);
16
+
17
+ return testConnectionCommand.description('Test connection').action(async (options) => {
18
+ try {
19
+ const config = await utils.loadConfig(extractRunnerOptions(options));
20
+ const serviceContext = new system.ServiceContextContainer(config);
21
+
22
+ const replication = new ReplicationEngine();
23
+ serviceContext.register(ReplicationEngine, replication);
24
+
25
+ // Register modules in order to load the correct config
26
+ const moduleManager = container.getImplementation(modules.ModuleManager);
27
+ await moduleManager.initialize(serviceContext);
28
+
29
+ // Start the storage engine in order to create the appropriate BucketStorage
30
+ await serviceContext.lifeCycleEngine.start();
31
+
32
+ logger.info('Testing connection...');
33
+ const results = await replication.testConnection();
34
+ logger.info(`Connection succeeded to ${results.map((r) => r.connectionDescription).join(', ')}`);
35
+ process.exit(0);
36
+ } catch (e) {
37
+ logger.error(`Connection failed: ${e.message}`);
38
+ process.exit(1);
39
+ }
40
+ });
41
+ }
@@ -3,7 +3,7 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
3
3
  import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
4
4
  import { Resource } from '@opentelemetry/resources';
5
5
  import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
6
- import { logger } from '@powersync/lib-services-framework';
6
+ import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
7
7
  import * as storage from '../storage/storage-index.js';
8
8
  import * as util from '../util/util-index.js';
9
9
 
@@ -132,7 +132,7 @@ export class Metrics {
132
132
 
133
133
  public static getInstance(): Metrics {
134
134
  if (!Metrics.instance) {
135
- throw new Error('Metrics have not been initialised');
135
+ throw new ServiceAssertionError('Metrics have not been initialized');
136
136
  }
137
137
 
138
138
  return Metrics.instance;
@@ -6,6 +6,7 @@ import { StorageEngine } from '../storage/storage-index.js';
6
6
  import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
7
7
  import { AbstractReplicationJob } from './AbstractReplicationJob.js';
8
8
  import { ErrorRateLimiter } from './ErrorRateLimiter.js';
9
+ import { ConnectionTestResult } from './ReplicationModule.js';
9
10
 
10
11
  // 5 minutes
11
12
  const PING_INTERVAL = 1_000_000_000n * 300n;
@@ -92,21 +93,27 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
92
93
 
93
94
  private async runLoop() {
94
95
  const syncRules = await this.syncRuleProvider.get();
96
+
95
97
  let configuredLock: storage.ReplicationLock | undefined = undefined;
96
98
  if (syncRules != null) {
97
99
  this.logger.info('Loaded sync rules');
98
100
  try {
99
101
  // Configure new sync rules, if they have changed.
100
102
  // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
101
- const { lock } = await this.storage.configureSyncRules(syncRules, {
102
- lock: true
103
+
104
+ const { lock } = await this.storage.configureSyncRules({
105
+ content: syncRules,
106
+ lock: true,
107
+ validate: this.syncRuleProvider.exitOnError
103
108
  });
104
109
  if (lock) {
105
110
  configuredLock = lock;
106
111
  }
107
112
  } catch (e) {
108
- // Log, but continue with previous sync rules
113
+ // Log and re-raise to exit.
114
+ // Should only reach this due to validation errors if exit_on_error is true.
109
115
  this.logger.error(`Failed to update sync rules from configuration`, e);
116
+ throw e;
110
117
  }
111
118
  } else {
112
119
  this.logger.info('No sync rules configured - configure via API');
@@ -225,4 +232,6 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
225
232
  this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
226
233
  }
227
234
  }
235
+
236
+ abstract testConnection(): Promise<ConnectionTestResult>;
228
237
  }
@@ -1,5 +1,6 @@
1
1
  import { logger } from '@powersync/lib-services-framework';
2
2
  import { AbstractReplicator } from './AbstractReplicator.js';
3
+ import { ConnectionTestResult } from './ReplicationModule.js';
3
4
 
4
5
  export class ReplicationEngine {
5
6
  private readonly replicators: Map<string, AbstractReplicator> = new Map();
@@ -40,4 +41,8 @@ export class ReplicationEngine {
40
41
  }
41
42
  logger.info('Successfully shut down Replication Engine.');
42
43
  }
44
+
45
+ public async testConnection(): Promise<ConnectionTestResult[]> {
46
+ return await Promise.all([...this.replicators.values()].map((replicator) => replicator.testConnection()));
47
+ }
43
48
  }
@@ -1,13 +1,19 @@
1
1
  import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
2
2
  import * as t from 'ts-codec';
3
3
 
4
+ import { schema } from '@powersync/lib-services-framework';
4
5
  import * as types from '@powersync/service-types';
5
6
  import * as api from '../api/api-index.js';
6
7
  import * as modules from '../modules/modules-index.js';
7
8
  import * as system from '../system/system-index.js';
8
- import { schema } from '@powersync/lib-services-framework';
9
9
  import { AbstractReplicator } from './AbstractReplicator.js';
10
- import { TearDownOptions } from '../modules/modules-index.js';
10
+
11
+ export interface ConnectionTestResult {
12
+ /**
13
+ * Connection URI or hostname.
14
+ */
15
+ connectionDescription: string;
16
+ }
11
17
 
12
18
  /**
13
19
  * Provides a common interface for testing the connection to a DataSource.
@@ -17,7 +23,7 @@ export interface ConnectionTester<TConfig extends DataSourceConfig> {
17
23
  * Confirm if a connection can be established to the datasource for the provided datasource configuration
18
24
  * @param config
19
25
  */
20
- testConnection(config: TConfig): Promise<void>;
26
+ testConnection(config: TConfig): Promise<ConnectionTestResult>;
21
27
  }
22
28
 
23
29
  export interface ReplicationModuleOptions extends modules.AbstractModuleOptions {
@@ -58,7 +64,7 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig>
58
64
  */
59
65
  protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
60
66
 
61
- public abstract testConnection(config: TConfig): Promise<void>;
67
+ public abstract testConnection(config: TConfig): Promise<ConnectionTestResult>;
62
68
 
63
69
  /**
64
70
  * Register this module's Replicators and RouteAPI adapters if the required configuration is present.
@@ -81,16 +87,12 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig>
81
87
  );
82
88
  }
83
89
 
84
- try {
85
- const baseMatchingConfig = matchingConfig[0] as TConfig;
86
- // If decoding fails, log the error and continue, no replication will happen for this data source
87
- this.decodeConfig(baseMatchingConfig);
90
+ const baseMatchingConfig = matchingConfig[0] as TConfig;
91
+ // If decoding fails, this will raise a hard error, and stop the service.
92
+ this.decodeConfig(baseMatchingConfig);
88
93
 
89
- context.replicationEngine?.register(this.createReplicator(context));
90
- context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
91
- } catch (e) {
92
- this.logger.error('Failed to initialize.', e);
93
- }
94
+ context.replicationEngine?.register(this.createReplicator(context));
95
+ context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
94
96
  }
95
97
 
96
98
  protected decodeConfig(config: TConfig): void {
@@ -1,4 +1,4 @@
1
- import { errors, router, schema } from '@powersync/lib-services-framework';
1
+ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
2
2
  import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
3
3
  import { internal_routes } from '@powersync/service-types';
4
4
 
@@ -120,15 +120,18 @@ export const reprocess = routeDefinition({
120
120
 
121
121
  const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
122
122
  if (active == null) {
123
- throw new errors.JourneyError({
123
+ throw new errors.ServiceError({
124
124
  status: 422,
125
- code: 'NO_SYNC_RULES',
125
+ code: ErrorCode.PSYNC_S4104,
126
126
  description: 'No active sync rules'
127
127
  });
128
128
  }
129
129
 
130
130
  const new_rules = await activeBucketStorage.updateSyncRules({
131
- content: active.sync_rules.content
131
+ content: active.sync_rules.content,
132
+ // These sync rules already passed validation. But if the rules are not valid anymore due
133
+ // to a service change, we do want to report the error here.
134
+ validate: true
132
135
  });
133
136
 
134
137
  const baseConfig = await apiHandler.getSourceConfig();
@@ -25,7 +25,7 @@ export const writeCheckpoint = routeDefinition({
25
25
  // Since we don't use LSNs anymore, the only way to get that is to wait.
26
26
  const start = Date.now();
27
27
 
28
- const head = await apiHandler.getReplicationHead();
28
+ const head = await apiHandler.createReplicationHead(async (head) => head);
29
29
 
30
30
  const timeout = 50_000;
31
31
 
@@ -56,25 +56,14 @@ export const writeCheckpoint2 = routeDefinition({
56
56
 
57
57
  const apiHandler = service_context.routerEngine!.getAPI();
58
58
 
59
- const client_id = payload.params.client_id;
60
- const full_user_id = util.checkpointUserId(user_id, client_id);
61
-
62
- const currentCheckpoint = await apiHandler.getReplicationHead();
63
- const {
64
- storageEngine: { activeBucketStorage }
65
- } = service_context;
66
-
67
- const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
68
- if (!activeSyncRules) {
69
- throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
70
- }
71
-
72
- using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
73
- const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
74
- user_id: full_user_id,
75
- heads: { '1': currentCheckpoint }
59
+ const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
60
+ userId: user_id,
61
+ clientId: payload.params.client_id,
62
+ api: apiHandler,
63
+ storage: service_context.storageEngine.activeBucketStorage
76
64
  });
77
- logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
65
+
66
+ logger.info(`Write checkpoint for ${user_id}/${payload.params.client_id}: ${writeCheckpoint} | ${replicationHead}`);
78
67
 
79
68
  return {
80
69
  write_checkpoint: String(writeCheckpoint)
@@ -1,4 +1,4 @@
1
- import { errors, logger, schema } from '@powersync/lib-services-framework';
1
+ import { ErrorCode, errors, logger, schema } from '@powersync/lib-services-framework';
2
2
  import { RequestParameters } from '@powersync/service-sync-rules';
3
3
  import { serialize } from 'bson';
4
4
 
@@ -34,9 +34,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
34
34
 
35
35
  if (routerEngine!.closed) {
36
36
  responder.onError(
37
- new errors.JourneyError({
37
+ new errors.ServiceError({
38
38
  status: 503,
39
- code: 'SERVICE_UNAVAILABLE',
39
+ code: ErrorCode.PSYNC_S2003,
40
40
  description: 'Service temporarily unavailable'
41
41
  })
42
42
  );
@@ -53,9 +53,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
53
53
  const cp = await activeBucketStorage.getActiveCheckpoint();
54
54
  if (!cp.hasSyncRules()) {
55
55
  responder.onError(
56
- new errors.JourneyError({
56
+ new errors.ServiceError({
57
57
  status: 500,
58
- code: 'NO_SYNC_RULES',
58
+ code: ErrorCode.PSYNC_S2302,
59
59
  description: 'No sync rules available'
60
60
  })
61
61
  );
@@ -1,4 +1,4 @@
1
- import { errors, router, schema } from '@powersync/lib-services-framework';
1
+ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
2
2
  import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
3
3
  import type { FastifyPluginAsync } from 'fastify';
4
4
  import * as t from 'ts-codec';
@@ -43,9 +43,9 @@ export const deploySyncRules = routeDefinition({
43
43
 
44
44
  if (service_context.configuration.sync_rules.present) {
45
45
  // If sync rules are configured via the config, disable deploy via the API.
46
- throw new errors.JourneyError({
46
+ throw new errors.ServiceError({
47
47
  status: 422,
48
- code: 'API_DISABLED',
48
+ code: ErrorCode.PSYNC_S4105,
49
49
  description: 'Sync rules API disabled',
50
50
  details: 'Use the management API to deploy sync rules'
51
51
  });
@@ -60,16 +60,18 @@ export const deploySyncRules = routeDefinition({
60
60
  schema: undefined
61
61
  });
62
62
  } catch (e) {
63
- throw new errors.JourneyError({
63
+ throw new errors.ServiceError({
64
64
  status: 422,
65
- code: 'INVALID_SYNC_RULES',
65
+ code: ErrorCode.PSYNC_R0001,
66
66
  description: 'Sync rules parsing failed',
67
67
  details: e.message
68
68
  });
69
69
  }
70
70
 
71
71
  const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
72
- content: content
72
+ content: content,
73
+ // Aready validated above
74
+ validate: false
73
75
  });
74
76
 
75
77
  return {
@@ -112,9 +114,9 @@ export const currentSyncRules = routeDefinition({
112
114
 
113
115
  const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
114
116
  if (!sync_rules) {
115
- throw new errors.JourneyError({
117
+ throw new errors.ServiceError({
116
118
  status: 422,
117
- code: 'NO_SYNC_RULES',
119
+ code: ErrorCode.PSYNC_S4104,
118
120
  description: 'No active sync rules'
119
121
  });
120
122
  }
@@ -159,15 +161,18 @@ export const reprocessSyncRules = routeDefinition({
159
161
  const apiHandler = payload.context.service_context.routerEngine!.getAPI();
160
162
  const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
161
163
  if (sync_rules == null) {
162
- throw new errors.JourneyError({
164
+ throw new errors.ServiceError({
163
165
  status: 422,
164
- code: 'NO_SYNC_RULES',
166
+ code: ErrorCode.PSYNC_S4104,
165
167
  description: 'No active sync rules'
166
168
  });
167
169
  }
168
170
 
169
171
  const new_rules = await activeBucketStorage.updateSyncRules({
170
- content: sync_rules.sync_rules.content
172
+ content: sync_rules.sync_rules.content,
173
+ // These sync rules already passed validation. But if the rules are not valid anymore due
174
+ // to a service change, we do want to report the error here.
175
+ validate: true
171
176
  });
172
177
  return {
173
178
  slot_name: new_rules.slot_name
@@ -1,4 +1,4 @@
1
- import { errors, logger, router, schema } from '@powersync/lib-services-framework';
1
+ import { ErrorCode, errors, logger, router, schema } from '@powersync/lib-services-framework';
2
2
  import { RequestParameters } from '@powersync/service-sync-rules';
3
3
  import { Readable } from 'stream';
4
4
 
@@ -26,9 +26,9 @@ export const syncStreamed = routeDefinition({
26
26
  const clientId = payload.params.client_id;
27
27
 
28
28
  if (routerEngine!.closed) {
29
- throw new errors.JourneyError({
29
+ throw new errors.ServiceError({
30
30
  status: 503,
31
- code: 'SERVICE_UNAVAILABLE',
31
+ code: ErrorCode.PSYNC_S2003,
32
32
  description: 'Service temporarily unavailable'
33
33
  });
34
34
  }
@@ -39,9 +39,9 @@ export const syncStreamed = routeDefinition({
39
39
  // Sanity check before we start the stream
40
40
  const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
41
41
  if (!cp.hasSyncRules()) {
42
- throw new errors.JourneyError({
42
+ throw new errors.ServiceError({
43
43
  status: 500,
44
- code: 'NO_SYNC_RULES',
44
+ code: ErrorCode.PSYNC_S2302,
45
45
  description: 'No sync rules available'
46
46
  });
47
47
  }
@@ -62,16 +62,16 @@ export function registerFastifyRoutes(
62
62
  });
63
63
  }
64
64
  } catch (ex) {
65
- const journeyError = errors.JourneyError.isJourneyError(ex) ? ex : new errors.InternalServerError(ex);
66
- logger.error(`Request failed`, journeyError);
65
+ const serviceError = errors.asServiceError(ex);
66
+ logger.error(`Request failed`, serviceError);
67
67
 
68
68
  response = new router.RouterResponse({
69
- status: journeyError.errorData.status || 500,
69
+ status: serviceError.errorData.status || 500,
70
70
  headers: {
71
71
  'Content-Type': 'application/json'
72
72
  },
73
73
  data: {
74
- error: journeyError.errorData
74
+ error: serviceError.errorData
75
75
  }
76
76
  });
77
77
  }
@@ -60,13 +60,26 @@ export interface BucketStorageFactoryListener extends DisposableListener {
60
60
  replicationEvent: (event: ReplicationEventPayload) => void;
61
61
  }
62
62
 
63
+ export interface BucketStorageSystemIdentifier {
64
+ /**
65
+ * A unique identifier for the system used for storage.
66
+ * For Postgres this can be the cluster `system_identifier` and database name.
67
+ * For MongoDB this can be the replica set name.
68
+ */
69
+ id: string;
70
+ /**
71
+ * A unique type for the storage implementation.
72
+ * e.g. `mongodb`, `postgresql`.
73
+ */
74
+ type: string;
75
+ }
76
+
63
77
  export interface BucketStorageFactory extends AsyncDisposableObserverClient<BucketStorageFactoryListener> {
64
78
  /**
65
79
  * Update sync rules from configuration, if changed.
66
80
  */
67
81
  configureSyncRules(
68
- sync_rules: string,
69
- options?: { lock?: boolean }
82
+ options: UpdateSyncRulesOptions
70
83
  ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>;
71
84
 
72
85
  /**
@@ -76,6 +89,8 @@ export interface BucketStorageFactory extends AsyncDisposableObserverClient<Buck
76
89
 
77
90
  /**
78
91
  * Deploy new sync rules.
92
+ *
93
+ * Similar to configureSyncRules, but applies the update unconditionally.
79
94
  */
80
95
  updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
81
96
 
@@ -143,6 +158,11 @@ export interface BucketStorageFactory extends AsyncDisposableObserverClient<Buck
143
158
  * Get the unique identifier for this instance of Powersync
144
159
  */
145
160
  getPowerSyncInstanceId(): Promise<string>;
161
+
162
+ /**
163
+ * Get a unique identifier for the system used for storage.
164
+ */
165
+ getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
146
166
  }
147
167
 
148
168
  export interface ReplicationCheckpoint {
@@ -213,6 +233,7 @@ export interface PersistedSyncRules {
213
233
  export interface UpdateSyncRulesOptions {
214
234
  content: string;
215
235
  lock?: boolean;
236
+ validate?: boolean;
216
237
  }
217
238
 
218
239
  export interface SyncRulesBucketStorageOptions {
@@ -367,6 +388,20 @@ export interface BucketBatchStorageListener extends DisposableListener {
367
388
  replicationEvent: (payload: ReplicationEventPayload) => void;
368
389
  }
369
390
 
391
+ export interface BucketBatchCommitOptions {
392
+ /**
393
+ * Creates a new checkpoint even if there were no persisted operations.
394
+ * Defaults to true.
395
+ */
396
+ createEmptyCheckpoints?: boolean;
397
+ }
398
+
399
+ export type ResolvedBucketBatchCommitOptions = Required<BucketBatchCommitOptions>;
400
+
401
+ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
402
+ createEmptyCheckpoints: true
403
+ };
404
+
370
405
  export interface BucketStorageBatch extends DisposableObserverClient<BucketBatchStorageListener> {
371
406
  /**
372
407
  * Save an op, and potentially flush.
@@ -398,11 +433,11 @@ export interface BucketStorageBatch extends DisposableObserverClient<BucketBatch
398
433
  flush(): Promise<FlushedResult | null>;
399
434
 
400
435
  /**
401
- * Flush and commit any saved ops. This creates a new checkpoint.
436
+ * Flush and commit any saved ops. This creates a new checkpoint by default.
402
437
  *
403
438
  * Only call this after a transaction.
404
439
  */
405
- commit(lsn: string): Promise<boolean>;
440
+ commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
406
441
 
407
442
  /**
408
443
  * Advance the checkpoint LSN position, without any associated op.