@powersync/service-core 1.20.5 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/dist/api/RouteAPI.d.ts +3 -3
- package/dist/api/diagnostics.d.ts +1 -1
- package/dist/api/diagnostics.js +18 -2
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +1 -1
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +1 -1
- package/dist/replication/AbstractReplicationJob.js +1 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +6 -6
- package/dist/replication/AbstractReplicator.js +21 -21
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/endpoints/admin.js +7 -3
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +7 -7
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +2 -2
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +4 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +9 -9
- package/dist/storage/BucketStorage.js +9 -9
- package/dist/storage/BucketStorageFactory.d.ts +23 -18
- package/dist/storage/BucketStorageFactory.js +12 -11
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
- package/dist/storage/PersistedSyncRulesContent.js +10 -3
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/SourceTable.d.ts +3 -3
- package/dist/storage/SourceTable.js +3 -3
- package/dist/storage/StorageVersionConfig.d.ts +1 -1
- package/dist/storage/StorageVersionConfig.js +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
- package/dist/storage/SyncRulesBucketStorage.js +14 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
- package/dist/storage/WriteCheckpointAPI.js +1 -1
- package/dist/storage/bson.d.ts +0 -1
- package/dist/storage/bson.js +0 -4
- package/dist/storage/bson.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +2 -5
- package/dist/sync/BucketChecksumState.js +116 -57
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +44 -0
- package/dist/tracing/PerformanceTracer.js +102 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -0
- package/dist/tracing/TraceWriter.d.ts +22 -0
- package/dist/tracing/TraceWriter.js +63 -0
- package/dist/tracing/TraceWriter.js.map +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/config/compound-config-collector.d.ts +1 -1
- package/dist/util/config/compound-config-collector.js +2 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/protocol-types.d.ts +1 -1
- package/dist/util/protocol-types.js +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +3 -3
- package/src/api/diagnostics.ts +26 -3
- package/src/entry/commands/teardown-action.ts +1 -1
- package/src/index.ts +2 -0
- package/src/modules/AbstractModule.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +1 -1
- package/src/replication/AbstractReplicator.ts +23 -23
- package/src/routes/endpoints/admin.ts +7 -3
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +7 -7
- package/src/routes/endpoints/sync-stream.ts +2 -2
- package/src/runner/teardown.ts +4 -4
- package/src/storage/BucketStorage.ts +9 -9
- package/src/storage/BucketStorageFactory.ts +29 -22
- package/src/storage/PersistedSyncRulesContent.ts +12 -4
- package/src/storage/SourceTable.ts +3 -3
- package/src/storage/StorageVersionConfig.ts +1 -1
- package/src/storage/SyncRulesBucketStorage.ts +46 -7
- package/src/storage/WriteCheckpointAPI.ts +6 -6
- package/src/storage/bson.ts +0 -5
- package/src/sync/BucketChecksumState.ts +137 -73
- package/src/sync/sync.ts +1 -1
- package/src/tracing/PerformanceTracer.ts +126 -0
- package/src/tracing/TraceWriter.ts +67 -0
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/compound-config-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
- package/src/util/env.ts +1 -1
- package/src/util/protocol-types.ts +1 -1
- package/test/src/auth.test.ts +109 -1
- package/test/src/diagnostics.test.ts +151 -0
- package/test/src/sync/BucketChecksumState.test.ts +221 -65
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -32,19 +32,19 @@ export interface AbstractReplicatorOptions {
|
|
|
32
32
|
/**
|
|
33
33
|
* A replicator manages the mechanics for replicating data from a data source to a storage bucket.
|
|
34
34
|
* This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
|
|
35
|
-
* It also handles any changes to the sync
|
|
35
|
+
* It also handles any changes to the sync config.
|
|
36
36
|
*/
|
|
37
37
|
export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
|
|
38
38
|
protected logger: winston.Logger;
|
|
39
39
|
private lockAlerted: boolean = false;
|
|
40
40
|
/**
|
|
41
|
-
* Map of replication jobs by
|
|
42
|
-
* transitioning to a new
|
|
41
|
+
* Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
|
|
42
|
+
* transitioning to a new replication stream.
|
|
43
43
|
*/
|
|
44
44
|
private replicationJobs = new Map<number, T>();
|
|
45
45
|
|
|
46
46
|
/**
|
|
47
|
-
* Map of
|
|
47
|
+
* Map of replciation stream ids to promises that are clearing the replication stream.
|
|
48
48
|
*
|
|
49
49
|
* We primarily do this to keep track of what we're currently clearing, but don't currently
|
|
50
50
|
* use the Promise value.
|
|
@@ -68,8 +68,8 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
68
68
|
abstract createJob(options: CreateJobOptions): T;
|
|
69
69
|
|
|
70
70
|
/**
|
|
71
|
-
*
|
|
72
|
-
*
|
|
71
|
+
* Clean up any configuration or state for the specified replication stream on the datasource.
|
|
72
|
+
* Should be a no-op if the replication stream has already been cleared
|
|
73
73
|
*/
|
|
74
74
|
abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
|
|
75
75
|
|
|
@@ -100,7 +100,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
100
100
|
public async start(): Promise<void> {
|
|
101
101
|
this.abortController = new AbortController();
|
|
102
102
|
this.runLoop().catch((e) => {
|
|
103
|
-
this.logger.error('
|
|
103
|
+
this.logger.error('Fatal replication error', e);
|
|
104
104
|
container.reporter.captureException(e);
|
|
105
105
|
setTimeout(() => {
|
|
106
106
|
process.exit(1);
|
|
@@ -135,9 +135,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
135
135
|
|
|
136
136
|
let configuredLock: storage.ReplicationLock | undefined = undefined;
|
|
137
137
|
if (syncRules != null) {
|
|
138
|
-
this.logger.info('Loaded sync
|
|
138
|
+
this.logger.info('Loaded sync config');
|
|
139
139
|
try {
|
|
140
|
-
// Configure new sync
|
|
140
|
+
// Configure new sync config, if they have changed.
|
|
141
141
|
// In that case, also immediately take out a lock, so that another process doesn't start replication on it.
|
|
142
142
|
|
|
143
143
|
const { lock } = await this.storage.configureSyncRules(
|
|
@@ -149,11 +149,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
149
149
|
} catch (e) {
|
|
150
150
|
// Log and re-raise to exit.
|
|
151
151
|
// Should only reach this due to validation errors if exit_on_error is true.
|
|
152
|
-
this.logger.error(`Failed to update sync
|
|
152
|
+
this.logger.error(`Failed to update sync config`, e);
|
|
153
153
|
throw e;
|
|
154
154
|
}
|
|
155
155
|
} else {
|
|
156
|
-
this.logger.info('No sync rules configured - configure via API');
|
|
156
|
+
this.logger.info('No sync streams or rules configured - configure via API');
|
|
157
157
|
}
|
|
158
158
|
while (!this.stopped) {
|
|
159
159
|
await container.probes.touch();
|
|
@@ -206,7 +206,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
206
206
|
// Remove from the list. Next refresh call will restart the job.
|
|
207
207
|
existingJobs.delete(syncRules.id);
|
|
208
208
|
} else {
|
|
209
|
-
// New sync
|
|
209
|
+
// New sync config was found (or resume after restart)
|
|
210
210
|
try {
|
|
211
211
|
let lock: storage.ReplicationLock;
|
|
212
212
|
if (configuredLock?.sync_rules_id == syncRules.id) {
|
|
@@ -229,15 +229,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
229
229
|
} catch (e) {
|
|
230
230
|
if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
|
|
231
231
|
if (!this.lockAlerted) {
|
|
232
|
-
|
|
232
|
+
syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
|
|
233
233
|
this.lockAlerted = true;
|
|
234
234
|
}
|
|
235
235
|
} else {
|
|
236
|
-
// Could be a sync
|
|
236
|
+
// Could be a sync config parse error,
|
|
237
237
|
// for example from stricter validation that was added.
|
|
238
238
|
// This will be retried every couple of seconds.
|
|
239
|
-
// When new (valid) sync
|
|
240
|
-
|
|
239
|
+
// When new (valid) sync config is deployed and processed, this one be disabled.
|
|
240
|
+
syncRules.logger.error('Failed to start replication for new sync config', e);
|
|
241
241
|
}
|
|
242
242
|
}
|
|
243
243
|
}
|
|
@@ -246,7 +246,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
246
246
|
this.replicationJobs = newJobs;
|
|
247
247
|
this.activeReplicationJob = activeJob;
|
|
248
248
|
|
|
249
|
-
// Stop any orphaned jobs that no longer have
|
|
249
|
+
// Stop any orphaned jobs that no longer have a replication stream.
|
|
250
250
|
// Termination happens below
|
|
251
251
|
for (let job of existingJobs.values()) {
|
|
252
252
|
// Old - stop and clean up
|
|
@@ -254,11 +254,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
254
254
|
await job.stop();
|
|
255
255
|
} catch (e) {
|
|
256
256
|
// This will be retried
|
|
257
|
-
|
|
257
|
+
job.storage.logger.warn('Failed to stop old replication job', e);
|
|
258
258
|
}
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
-
//
|
|
261
|
+
// Replication stream stopped previously, including by a different process.
|
|
262
262
|
const stopped = await this.storage.getStoppedSyncRules();
|
|
263
263
|
for (let syncRules of stopped) {
|
|
264
264
|
if (this.clearingJobs.has(syncRules.id)) {
|
|
@@ -268,11 +268,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
268
268
|
|
|
269
269
|
// We clear storage asynchronously.
|
|
270
270
|
// It is important to be able to continue running the refresh loop, otherwise we cannot
|
|
271
|
-
// retry locked
|
|
271
|
+
// retry locked replication stream, for example.
|
|
272
272
|
const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
|
|
273
273
|
const promise = this.terminateSyncRules(syncRuleStorage)
|
|
274
274
|
.catch((e) => {
|
|
275
|
-
|
|
275
|
+
syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
|
|
276
276
|
})
|
|
277
277
|
.finally(() => {
|
|
278
278
|
this.clearingJobs.delete(syncRules.id);
|
|
@@ -286,12 +286,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
286
286
|
}
|
|
287
287
|
|
|
288
288
|
protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
|
|
289
|
-
|
|
289
|
+
syncRuleStorage.logger.info(`Terminating replication stream...`);
|
|
290
290
|
// This deletes postgres replication slots - should complete quickly.
|
|
291
291
|
// It is safe to do before or after clearing the data in the storage.
|
|
292
292
|
await this.cleanUp(syncRuleStorage);
|
|
293
293
|
await syncRuleStorage.terminate({ signal: this.abortController?.signal, clearStorage: true });
|
|
294
|
-
|
|
294
|
+
syncRuleStorage.logger.info(`Successfully terminated replication stream`);
|
|
295
295
|
}
|
|
296
296
|
|
|
297
297
|
abstract testConnection(): Promise<ConnectionTestResult>;
|
|
@@ -119,7 +119,7 @@ export const reprocess = routeDefinition({
|
|
|
119
119
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
120
120
|
const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
|
|
121
121
|
if (next != null) {
|
|
122
|
-
throw new Error(`Busy processing sync
|
|
122
|
+
throw new Error(`Busy processing sync config - cannot reprocess`);
|
|
123
123
|
}
|
|
124
124
|
|
|
125
125
|
const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
|
|
@@ -127,13 +127,17 @@ export const reprocess = routeDefinition({
|
|
|
127
127
|
throw new errors.ServiceError({
|
|
128
128
|
status: 422,
|
|
129
129
|
code: ErrorCode.PSYNC_S4104,
|
|
130
|
-
description: 'No active sync
|
|
130
|
+
description: 'No active sync config'
|
|
131
131
|
});
|
|
132
132
|
}
|
|
133
133
|
|
|
134
|
+
// There are some differences between this and using asUpdateOptions():
|
|
135
|
+
// 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
|
|
136
|
+
// 2. If the source does not set the storage version, this will update it do the current version.
|
|
137
|
+
// We can consider tweaking this behavior in the future.
|
|
134
138
|
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
135
139
|
storage.updateSyncRulesFromYaml(active.sync_rules.config.content, {
|
|
136
|
-
//
|
|
140
|
+
// This sync config already passed validation. But if the config is not valid anymore due
|
|
137
141
|
// to a service change, we do want to report the error here.
|
|
138
142
|
validate: true
|
|
139
143
|
})
|
|
@@ -33,7 +33,7 @@ export const writeCheckpoint = routeDefinition({
|
|
|
33
33
|
const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
|
|
34
34
|
const cp = await bucketStorage?.getCheckpoint();
|
|
35
35
|
if (cp == null) {
|
|
36
|
-
throw new Error('No sync
|
|
36
|
+
throw new Error('No sync config available');
|
|
37
37
|
}
|
|
38
38
|
if (cp.lsn && cp.lsn >= head) {
|
|
39
39
|
logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
|
|
@@ -77,7 +77,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
77
77
|
new errors.ServiceError({
|
|
78
78
|
status: 500,
|
|
79
79
|
code: ErrorCode.PSYNC_S2302,
|
|
80
|
-
description: 'No sync
|
|
80
|
+
description: 'No sync config available'
|
|
81
81
|
})
|
|
82
82
|
);
|
|
83
83
|
responder.onComplete();
|
|
@@ -43,12 +43,12 @@ export const deploySyncRules = routeDefinition({
|
|
|
43
43
|
const { storageEngine } = service_context;
|
|
44
44
|
|
|
45
45
|
if (service_context.configuration.sync_rules.present) {
|
|
46
|
-
// If sync
|
|
46
|
+
// If sync config is configured via the service config, disable deploy via the API.
|
|
47
47
|
throw new errors.ServiceError({
|
|
48
48
|
status: 422,
|
|
49
49
|
code: ErrorCode.PSYNC_S4105,
|
|
50
|
-
description: 'Sync
|
|
51
|
-
details: '
|
|
50
|
+
description: 'Sync config API disabled',
|
|
51
|
+
details: 'Update sync config in the service configuration'
|
|
52
52
|
});
|
|
53
53
|
}
|
|
54
54
|
const content = payload.params.content;
|
|
@@ -65,7 +65,7 @@ export const deploySyncRules = routeDefinition({
|
|
|
65
65
|
throw new errors.ServiceError({
|
|
66
66
|
status: 422,
|
|
67
67
|
code: ErrorCode.PSYNC_R0001,
|
|
68
|
-
description: 'Sync
|
|
68
|
+
description: 'Sync config parsing failed',
|
|
69
69
|
details: e.message
|
|
70
70
|
});
|
|
71
71
|
}
|
|
@@ -115,7 +115,7 @@ export const currentSyncRules = routeDefinition({
|
|
|
115
115
|
throw new errors.ServiceError({
|
|
116
116
|
status: 422,
|
|
117
117
|
code: ErrorCode.PSYNC_S4104,
|
|
118
|
-
description: 'No active sync
|
|
118
|
+
description: 'No active sync config'
|
|
119
119
|
});
|
|
120
120
|
}
|
|
121
121
|
|
|
@@ -162,13 +162,13 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
162
162
|
throw new errors.ServiceError({
|
|
163
163
|
status: 422,
|
|
164
164
|
code: ErrorCode.PSYNC_S4104,
|
|
165
|
-
description: 'No active sync
|
|
165
|
+
description: 'No active sync config'
|
|
166
166
|
});
|
|
167
167
|
}
|
|
168
168
|
|
|
169
169
|
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
170
170
|
updateSyncRulesFromYaml(sync_rules.sync_rules.config.content, {
|
|
171
|
-
//
|
|
171
|
+
// This sync config already passed validation. But if the rules are not valid anymore due
|
|
172
172
|
// to a service change, we do want to report the error here.
|
|
173
173
|
validate: true
|
|
174
174
|
})
|
|
@@ -68,7 +68,7 @@ export const syncStreamed = routeDefinition({
|
|
|
68
68
|
throw new errors.ServiceError({
|
|
69
69
|
status: 500,
|
|
70
70
|
code: ErrorCode.PSYNC_S2302,
|
|
71
|
-
description: 'No sync
|
|
71
|
+
description: 'No sync config available'
|
|
72
72
|
});
|
|
73
73
|
}
|
|
74
74
|
|
|
@@ -121,7 +121,7 @@ export const syncStreamed = routeDefinition({
|
|
|
121
121
|
});
|
|
122
122
|
|
|
123
123
|
stream.on('end', () => {
|
|
124
|
-
// Auth failure or switch to new sync
|
|
124
|
+
// Auth failure or switch to new sync config
|
|
125
125
|
closeReason ??= 'service closing stream';
|
|
126
126
|
});
|
|
127
127
|
|
package/src/runner/teardown.ts
CHANGED
|
@@ -35,13 +35,13 @@ export async function teardown(runnerConfig: utils.RunnerConfig) {
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, moduleManager: modules.ModuleManager) {
|
|
38
|
-
logger.info(`Terminating
|
|
38
|
+
logger.info(`Terminating replication stream...`);
|
|
39
39
|
const start = Date.now();
|
|
40
40
|
const locks: storage.ReplicationLock[] = [];
|
|
41
41
|
while (Date.now() - start < 120_000) {
|
|
42
42
|
let retry = false;
|
|
43
43
|
const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
|
|
44
|
-
// Lock all the replicating
|
|
44
|
+
// Lock all the replicating replication streams
|
|
45
45
|
for (const replicatingSyncRule of replicatingSyncRules) {
|
|
46
46
|
const lock = await replicatingSyncRule.lock();
|
|
47
47
|
locks.push(lock);
|
|
@@ -50,10 +50,10 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
|
|
|
50
50
|
const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
|
|
51
51
|
const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
|
|
52
52
|
try {
|
|
53
|
-
// Clean up any module specific configuration for the
|
|
53
|
+
// Clean up any module specific configuration for the replication stream
|
|
54
54
|
await moduleManager.tearDown({ syncRules: combinedSyncRules });
|
|
55
55
|
|
|
56
|
-
// Mark the
|
|
56
|
+
// Mark the replication stream as terminated
|
|
57
57
|
for (let syncRules of combinedSyncRules) {
|
|
58
58
|
const syncRulesStorage = storageFactory.getInstance(syncRules);
|
|
59
59
|
// The storage will be dropped at the end of the teardown, so we don't need to clear it here
|
|
@@ -2,36 +2,36 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
|
|
|
2
2
|
|
|
3
3
|
export enum SyncRuleState {
|
|
4
4
|
/**
|
|
5
|
-
* New
|
|
5
|
+
* New replication stream - needs to be processed (initial replication).
|
|
6
6
|
*
|
|
7
|
-
* While multiple
|
|
7
|
+
* While multiple replication streams _can_ be in PROCESSING,
|
|
8
8
|
* it's generally pointless, so we only keep one in that state.
|
|
9
9
|
*/
|
|
10
10
|
PROCESSING = 'PROCESSING',
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
|
-
*
|
|
13
|
+
* Intial processing is done, and can be used for sync.
|
|
14
14
|
*
|
|
15
|
-
* Only one
|
|
15
|
+
* Only one replication stream should be in ACTIVE or ERRORED state.
|
|
16
16
|
*/
|
|
17
17
|
ACTIVE = 'ACTIVE',
|
|
18
18
|
/**
|
|
19
|
-
* This state is used when the
|
|
19
|
+
* This state is used when the replication stream has been replaced,
|
|
20
20
|
* and replication is or should be stopped.
|
|
21
21
|
*/
|
|
22
22
|
STOP = 'STOP',
|
|
23
23
|
/**
|
|
24
|
-
* After
|
|
24
|
+
* After replication stream has been stopped, the data needs to be
|
|
25
25
|
* deleted. Once deleted, the state is TERMINATED.
|
|
26
26
|
*/
|
|
27
27
|
TERMINATED = 'TERMINATED',
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
|
-
*
|
|
31
|
-
* is still the "active"
|
|
30
|
+
* Replication stream has run into a permanent replication error. It
|
|
31
|
+
* is still the "active" replication stram for syncing to users,
|
|
32
32
|
* but should not replicate anymore.
|
|
33
33
|
*
|
|
34
|
-
* It will transition to STOP when a new
|
|
34
|
+
* It will transition to STOP when a new replication stream is activated.
|
|
35
35
|
*/
|
|
36
36
|
ERRORED = 'ERRORED'
|
|
37
37
|
}
|
|
@@ -17,17 +17,17 @@ import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
|
|
|
17
17
|
/**
|
|
18
18
|
* Represents a configured storage provider.
|
|
19
19
|
*
|
|
20
|
-
* The provider can handle multiple
|
|
21
|
-
* This is to handle replication of a new version of sync
|
|
20
|
+
* The provider can handle multiple replication streams concurrently, each with their own storage.
|
|
21
|
+
* This is to handle replication of a new version of sync config, while the old replication stream is still active.
|
|
22
22
|
*
|
|
23
|
-
* Storage APIs for a specific
|
|
23
|
+
* Storage APIs for a specific replication stream are provided by the `SyncRulesBucketStorage` instances.
|
|
24
24
|
*/
|
|
25
25
|
export abstract class BucketStorageFactory
|
|
26
26
|
extends BaseObserver<BucketStorageFactoryListener>
|
|
27
27
|
implements AsyncDisposable
|
|
28
28
|
{
|
|
29
29
|
/**
|
|
30
|
-
* Update sync
|
|
30
|
+
* Update sync config from configuration, if changed.
|
|
31
31
|
*/
|
|
32
32
|
async configureSyncRules(
|
|
33
33
|
options: UpdateSyncRulesOptions
|
|
@@ -36,42 +36,42 @@ export abstract class BucketStorageFactory
|
|
|
36
36
|
const active = await this.getActiveSyncRulesContent();
|
|
37
37
|
|
|
38
38
|
if (next?.sync_rules_content == options.config.yaml) {
|
|
39
|
-
logger.info('Sync
|
|
39
|
+
logger.info('Sync config unchanged');
|
|
40
40
|
return { updated: false };
|
|
41
41
|
} else if (next == null && active?.sync_rules_content == options.config.yaml) {
|
|
42
|
-
logger.info('Sync
|
|
42
|
+
logger.info('Sync config unchanged');
|
|
43
43
|
return { updated: false };
|
|
44
44
|
} else {
|
|
45
|
-
logger.info('Sync
|
|
45
|
+
logger.info('Sync config updated');
|
|
46
46
|
const persisted_sync_rules = await this.updateSyncRules(options);
|
|
47
47
|
return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
|
-
* Get a storage instance to query sync data for specific sync
|
|
52
|
+
* Get a storage instance to query sync data for specific sync config.
|
|
53
53
|
*/
|
|
54
54
|
abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
|
-
* Deploy new sync
|
|
57
|
+
* Deploy new sync config.
|
|
58
58
|
*/
|
|
59
59
|
abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
62
|
* Indicate that a slot was removed, and we should re-sync by creating
|
|
63
|
-
* a new
|
|
63
|
+
* a new replication stream.
|
|
64
64
|
*
|
|
65
65
|
* This is roughly the same as deploying a new version of the current sync
|
|
66
|
-
*
|
|
67
|
-
* the latest
|
|
66
|
+
* config, but also accounts for cases where the current sync config is not
|
|
67
|
+
* the latest one.
|
|
68
68
|
*
|
|
69
69
|
* Replication should be restarted after this.
|
|
70
70
|
*/
|
|
71
71
|
abstract restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
* Get the sync
|
|
74
|
+
* Get the sync config used for querying.
|
|
75
75
|
*/
|
|
76
76
|
async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
77
77
|
const content = await this.getActiveSyncRulesContent();
|
|
@@ -79,12 +79,12 @@ export abstract class BucketStorageFactory
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
/**
|
|
82
|
-
* Get the sync
|
|
82
|
+
* Get the sync config used for querying.
|
|
83
83
|
*/
|
|
84
84
|
abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
85
85
|
|
|
86
86
|
/**
|
|
87
|
-
* Get the sync
|
|
87
|
+
* Get the sync config that will be active next once done with initial replicatino.
|
|
88
88
|
*/
|
|
89
89
|
async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
90
90
|
const content = await this.getNextSyncRulesContent();
|
|
@@ -92,17 +92,17 @@ export abstract class BucketStorageFactory
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
/**
|
|
95
|
-
* Get the sync
|
|
95
|
+
* Get the sync config that will be active next once done with initial replicatino.
|
|
96
96
|
*/
|
|
97
97
|
abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
98
98
|
|
|
99
99
|
/**
|
|
100
|
-
* Get all sync
|
|
100
|
+
* Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
|
|
101
101
|
*/
|
|
102
102
|
abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
103
103
|
|
|
104
104
|
/**
|
|
105
|
-
* Get all sync
|
|
105
|
+
* Get all sync config stopped but not terminated yet.
|
|
106
106
|
*/
|
|
107
107
|
abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
108
108
|
|
|
@@ -112,12 +112,12 @@ export abstract class BucketStorageFactory
|
|
|
112
112
|
abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
113
113
|
|
|
114
114
|
/**
|
|
115
|
-
* Get storage size of active
|
|
115
|
+
* Get storage size of active replication stream.
|
|
116
116
|
*/
|
|
117
117
|
abstract getStorageMetrics(): Promise<StorageMetrics>;
|
|
118
118
|
|
|
119
119
|
/**
|
|
120
|
-
* Get the unique identifier for this instance of Powersync
|
|
120
|
+
* Get the unique identifier for this instance of Powersync.
|
|
121
121
|
*/
|
|
122
122
|
abstract getPowerSyncInstanceId(): Promise<string>;
|
|
123
123
|
|
|
@@ -161,6 +161,12 @@ export interface UpdateSyncRulesOptions {
|
|
|
161
161
|
* compiler.
|
|
162
162
|
*/
|
|
163
163
|
plan: SerializedSyncPlan | null;
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Parsed sync config, primarily to generate a definition mapping.
|
|
167
|
+
* Not persisted, and the defaultSchema used for parsing is not relevant.
|
|
168
|
+
*/
|
|
169
|
+
parsed: SyncConfigWithErrors;
|
|
164
170
|
};
|
|
165
171
|
lock?: boolean;
|
|
166
172
|
storageVersion?: number;
|
|
@@ -198,10 +204,11 @@ export function updateSyncRulesFromYaml(
|
|
|
198
204
|
}
|
|
199
205
|
|
|
200
206
|
export function updateSyncRulesFromConfig(
|
|
201
|
-
|
|
207
|
+
parsed: SyncConfigWithErrors,
|
|
202
208
|
options?: Omit<UpdateSyncRulesOptions, 'config'>
|
|
203
209
|
): UpdateSyncRulesOptions {
|
|
204
210
|
let plan: SerializedSyncPlan | null = null;
|
|
211
|
+
const { config, errors } = parsed;
|
|
205
212
|
if (config instanceof PrecompiledSyncConfig) {
|
|
206
213
|
const eventDescriptors: Record<string, string[]> = {};
|
|
207
214
|
for (const event of config.eventDescriptors) {
|
|
@@ -216,7 +223,7 @@ export function updateSyncRulesFromConfig(
|
|
|
216
223
|
};
|
|
217
224
|
}
|
|
218
225
|
|
|
219
|
-
return { config: { yaml: config.content, plan }, ...options };
|
|
226
|
+
return { config: { yaml: config.content, plan, parsed }, ...options };
|
|
220
227
|
}
|
|
221
228
|
|
|
222
229
|
export interface GetIntanceOptions {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
1
|
+
import { logger as defaultLogger, ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
|
|
2
2
|
import {
|
|
3
3
|
CompatibilityContext,
|
|
4
4
|
CompatibilityOption,
|
|
@@ -29,7 +29,7 @@ export interface PersistedSyncRulesContentData {
|
|
|
29
29
|
readonly compiled_plan: SerializedSyncPlan | null;
|
|
30
30
|
readonly slot_name: string;
|
|
31
31
|
/**
|
|
32
|
-
* True if this is the "active" copy of the sync
|
|
32
|
+
* True if this is the "active" copy of the sync config.
|
|
33
33
|
*/
|
|
34
34
|
readonly active: boolean;
|
|
35
35
|
readonly storageVersion: number;
|
|
@@ -49,6 +49,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
|
|
|
49
49
|
readonly slot_name!: string;
|
|
50
50
|
readonly active!: boolean;
|
|
51
51
|
readonly storageVersion!: number;
|
|
52
|
+
readonly logger: Logger;
|
|
52
53
|
|
|
53
54
|
readonly last_checkpoint_lsn!: string | null;
|
|
54
55
|
|
|
@@ -61,6 +62,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
|
|
|
61
62
|
|
|
62
63
|
constructor(data: PersistedSyncRulesContentData) {
|
|
63
64
|
Object.assign(this, data);
|
|
65
|
+
this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` });
|
|
64
66
|
}
|
|
65
67
|
|
|
66
68
|
/**
|
|
@@ -73,7 +75,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
|
|
|
73
75
|
if (storageConfig == null) {
|
|
74
76
|
throw new ServiceError(
|
|
75
77
|
ErrorCode.PSYNC_S1005,
|
|
76
|
-
`Unsupported storage version ${this.storageVersion} for
|
|
78
|
+
`Unsupported storage version ${this.storageVersion} for replication stream ${this.id}`
|
|
77
79
|
);
|
|
78
80
|
}
|
|
79
81
|
return storageConfig;
|
|
@@ -103,6 +105,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
|
|
|
103
105
|
sourceText: this.sync_rules_content
|
|
104
106
|
});
|
|
105
107
|
|
|
108
|
+
// Note: If the original content did not define a storage version, this will still set the storage version.
|
|
109
|
+
// This means asUpdateOptions will not change the storage version, even if the default changes.
|
|
110
|
+
precompiled.storageVersion = this.storageVersion;
|
|
111
|
+
|
|
106
112
|
const errors: YamlError[] = [];
|
|
107
113
|
if (this.compiled_plan.errors) {
|
|
108
114
|
for (const error of this.compiled_plan.errors) {
|
|
@@ -144,8 +150,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
|
|
|
144
150
|
}
|
|
145
151
|
|
|
146
152
|
asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
|
|
153
|
+
// defaultSchema is not relevant for the parsed version here
|
|
154
|
+
const parsed = this.parsed({ defaultSchema: 'not_applicable' });
|
|
147
155
|
return {
|
|
148
|
-
config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
|
|
156
|
+
config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules },
|
|
149
157
|
...options
|
|
150
158
|
};
|
|
151
159
|
}
|
|
@@ -28,7 +28,7 @@ export class SourceTable implements SourceEntityDescriptor {
|
|
|
28
28
|
static readonly DEFAULT_TAG = DEFAULT_TAG;
|
|
29
29
|
|
|
30
30
|
/**
|
|
31
|
-
* True if the table is used in sync
|
|
31
|
+
* True if the table is used in sync config for data queries.
|
|
32
32
|
*
|
|
33
33
|
* This value is resolved externally, and cached here.
|
|
34
34
|
*
|
|
@@ -37,7 +37,7 @@ export class SourceTable implements SourceEntityDescriptor {
|
|
|
37
37
|
public syncData = true;
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
|
-
* True if the table is used in sync
|
|
40
|
+
* True if the table is used in sync config for data queries.
|
|
41
41
|
*
|
|
42
42
|
* This value is resolved externally, and cached here.
|
|
43
43
|
*
|
|
@@ -46,7 +46,7 @@ export class SourceTable implements SourceEntityDescriptor {
|
|
|
46
46
|
public syncParameters = true;
|
|
47
47
|
|
|
48
48
|
/**
|
|
49
|
-
* True if the table is used in sync
|
|
49
|
+
* True if the table is used in sync config for events.
|
|
50
50
|
*
|
|
51
51
|
* This value is resolved externally, and cached here.
|
|
52
52
|
*
|
|
@@ -45,7 +45,7 @@ export const STORAGE_VERSION_3 = 3;
|
|
|
45
45
|
export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
|
-
* Default storage version for newly persisted
|
|
48
|
+
* Default storage version for newly persisted replication streams.
|
|
49
49
|
*/
|
|
50
50
|
export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
|
|
51
51
|
|