@powersync/service-core 1.20.5 → 1.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/dist/api/RouteAPI.d.ts +3 -3
  3. package/dist/api/diagnostics.d.ts +1 -1
  4. package/dist/api/diagnostics.js +18 -2
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/teardown-action.js +1 -1
  7. package/dist/entry/commands/teardown-action.js.map +1 -1
  8. package/dist/index.d.ts +1 -0
  9. package/dist/index.js +1 -0
  10. package/dist/index.js.map +1 -1
  11. package/dist/modules/AbstractModule.d.ts +1 -1
  12. package/dist/replication/AbstractReplicationJob.js +1 -1
  13. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  14. package/dist/replication/AbstractReplicator.d.ts +6 -6
  15. package/dist/replication/AbstractReplicator.js +21 -21
  16. package/dist/replication/AbstractReplicator.js.map +1 -1
  17. package/dist/routes/endpoints/admin.js +7 -3
  18. package/dist/routes/endpoints/admin.js.map +1 -1
  19. package/dist/routes/endpoints/checkpointing.js +1 -1
  20. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  21. package/dist/routes/endpoints/socket-route.js +1 -1
  22. package/dist/routes/endpoints/socket-route.js.map +1 -1
  23. package/dist/routes/endpoints/sync-rules.js +7 -7
  24. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  25. package/dist/routes/endpoints/sync-stream.js +2 -2
  26. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  27. package/dist/runner/teardown.js +4 -4
  28. package/dist/runner/teardown.js.map +1 -1
  29. package/dist/storage/BucketStorage.d.ts +9 -9
  30. package/dist/storage/BucketStorage.js +9 -9
  31. package/dist/storage/BucketStorageFactory.d.ts +23 -18
  32. package/dist/storage/BucketStorageFactory.js +12 -11
  33. package/dist/storage/BucketStorageFactory.js.map +1 -1
  34. package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
  35. package/dist/storage/PersistedSyncRulesContent.js +10 -3
  36. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  37. package/dist/storage/SourceTable.d.ts +3 -3
  38. package/dist/storage/SourceTable.js +3 -3
  39. package/dist/storage/StorageVersionConfig.d.ts +1 -1
  40. package/dist/storage/StorageVersionConfig.js +1 -1
  41. package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
  42. package/dist/storage/SyncRulesBucketStorage.js +14 -0
  43. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  44. package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
  45. package/dist/storage/WriteCheckpointAPI.js +1 -1
  46. package/dist/storage/bson.d.ts +0 -1
  47. package/dist/storage/bson.js +0 -4
  48. package/dist/storage/bson.js.map +1 -1
  49. package/dist/sync/BucketChecksumState.d.ts +2 -5
  50. package/dist/sync/BucketChecksumState.js +116 -57
  51. package/dist/sync/BucketChecksumState.js.map +1 -1
  52. package/dist/tracing/PerformanceTracer.d.ts +44 -0
  53. package/dist/tracing/PerformanceTracer.js +102 -0
  54. package/dist/tracing/PerformanceTracer.js.map +1 -0
  55. package/dist/tracing/TraceWriter.d.ts +22 -0
  56. package/dist/tracing/TraceWriter.js +63 -0
  57. package/dist/tracing/TraceWriter.js.map +1 -0
  58. package/dist/util/checkpointing.js +1 -1
  59. package/dist/util/config/compound-config-collector.d.ts +1 -1
  60. package/dist/util/config/compound-config-collector.js +2 -2
  61. package/dist/util/config/compound-config-collector.js.map +1 -1
  62. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
  63. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  64. package/dist/util/env.js +1 -1
  65. package/dist/util/protocol-types.d.ts +1 -1
  66. package/dist/util/protocol-types.js +1 -1
  67. package/package.json +11 -11
  68. package/src/api/RouteAPI.ts +3 -3
  69. package/src/api/diagnostics.ts +26 -3
  70. package/src/entry/commands/teardown-action.ts +1 -1
  71. package/src/index.ts +2 -0
  72. package/src/modules/AbstractModule.ts +1 -1
  73. package/src/replication/AbstractReplicationJob.ts +1 -1
  74. package/src/replication/AbstractReplicator.ts +23 -23
  75. package/src/routes/endpoints/admin.ts +7 -3
  76. package/src/routes/endpoints/checkpointing.ts +1 -1
  77. package/src/routes/endpoints/socket-route.ts +1 -1
  78. package/src/routes/endpoints/sync-rules.ts +7 -7
  79. package/src/routes/endpoints/sync-stream.ts +2 -2
  80. package/src/runner/teardown.ts +4 -4
  81. package/src/storage/BucketStorage.ts +9 -9
  82. package/src/storage/BucketStorageFactory.ts +29 -22
  83. package/src/storage/PersistedSyncRulesContent.ts +12 -4
  84. package/src/storage/SourceTable.ts +3 -3
  85. package/src/storage/StorageVersionConfig.ts +1 -1
  86. package/src/storage/SyncRulesBucketStorage.ts +46 -7
  87. package/src/storage/WriteCheckpointAPI.ts +6 -6
  88. package/src/storage/bson.ts +0 -5
  89. package/src/sync/BucketChecksumState.ts +137 -73
  90. package/src/sync/sync.ts +1 -1
  91. package/src/tracing/PerformanceTracer.ts +126 -0
  92. package/src/tracing/TraceWriter.ts +67 -0
  93. package/src/util/checkpointing.ts +1 -1
  94. package/src/util/config/compound-config-collector.ts +3 -3
  95. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
  96. package/src/util/env.ts +1 -1
  97. package/src/util/protocol-types.ts +1 -1
  98. package/test/src/auth.test.ts +109 -1
  99. package/test/src/diagnostics.test.ts +151 -0
  100. package/test/src/sync/BucketChecksumState.test.ts +221 -65
  101. package/test/tsconfig.json +0 -1
  102. package/tsconfig.tsbuildinfo +1 -1
@@ -32,19 +32,19 @@ export interface AbstractReplicatorOptions {
32
32
  /**
33
33
  * A replicator manages the mechanics for replicating data from a data source to a storage bucket.
34
34
  * This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
35
- * It also handles any changes to the sync rules.
35
+ * It also handles any changes to the sync config.
36
36
  */
37
37
  export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
38
38
  protected logger: winston.Logger;
39
39
  private lockAlerted: boolean = false;
40
40
  /**
41
- * Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
42
- * transitioning to a new set of sync rules.
41
+ * Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
42
+ * transitioning to a new replication stream.
43
43
  */
44
44
  private replicationJobs = new Map<number, T>();
45
45
 
46
46
  /**
47
- * Map of sync rule ids to promises that are clearing the sync rule configuration.
47
+ * Map of replciation stream ids to promises that are clearing the replication stream.
48
48
  *
49
49
  * We primarily do this to keep track of what we're currently clearing, but don't currently
50
50
  * use the Promise value.
@@ -68,8 +68,8 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
68
68
  abstract createJob(options: CreateJobOptions): T;
69
69
 
70
70
  /**
71
- * Clean up any configuration or state for the specified sync rule on the datasource.
72
- * Should be a no-op if the configuration has already been cleared
71
+ * Clean up any configuration or state for the specified replication stream on the datasource.
72
+ * Should be a no-op if the replication stream has already been cleared
73
73
  */
74
74
  abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
75
75
 
@@ -100,7 +100,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
100
100
  public async start(): Promise<void> {
101
101
  this.abortController = new AbortController();
102
102
  this.runLoop().catch((e) => {
103
- this.logger.error('Data source fatal replication error', e);
103
+ this.logger.error('Fatal replication error', e);
104
104
  container.reporter.captureException(e);
105
105
  setTimeout(() => {
106
106
  process.exit(1);
@@ -135,9 +135,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
135
135
 
136
136
  let configuredLock: storage.ReplicationLock | undefined = undefined;
137
137
  if (syncRules != null) {
138
- this.logger.info('Loaded sync rules');
138
+ this.logger.info('Loaded sync config');
139
139
  try {
140
- // Configure new sync rules, if they have changed.
140
+ // Configure new sync config, if they have changed.
141
141
  // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
142
142
 
143
143
  const { lock } = await this.storage.configureSyncRules(
@@ -149,11 +149,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
149
149
  } catch (e) {
150
150
  // Log and re-raise to exit.
151
151
  // Should only reach this due to validation errors if exit_on_error is true.
152
- this.logger.error(`Failed to update sync rules from configuration`, e);
152
+ this.logger.error(`Failed to update sync config`, e);
153
153
  throw e;
154
154
  }
155
155
  } else {
156
- this.logger.info('No sync rules configured - configure via API');
156
+ this.logger.info('No sync streams or rules configured - configure via API');
157
157
  }
158
158
  while (!this.stopped) {
159
159
  await container.probes.touch();
@@ -206,7 +206,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
206
206
  // Remove from the list. Next refresh call will restart the job.
207
207
  existingJobs.delete(syncRules.id);
208
208
  } else {
209
- // New sync rules were found (or resume after restart)
209
+ // New sync config was found (or resume after restart)
210
210
  try {
211
211
  let lock: storage.ReplicationLock;
212
212
  if (configuredLock?.sync_rules_id == syncRules.id) {
@@ -229,15 +229,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
229
229
  } catch (e) {
230
230
  if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
231
231
  if (!this.lockAlerted) {
232
- this.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
232
+ syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
233
233
  this.lockAlerted = true;
234
234
  }
235
235
  } else {
236
- // Could be a sync rules parse error,
236
+ // Could be a sync config parse error,
237
237
  // for example from stricter validation that was added.
238
238
  // This will be retried every couple of seconds.
239
- // When new (valid) sync rules are deployed and processed, this one be disabled.
240
- this.logger.error('Failed to start replication for new sync rules', e);
239
+ // When new (valid) sync config is deployed and processed, this one be disabled.
240
+ syncRules.logger.error('Failed to start replication for new sync config', e);
241
241
  }
242
242
  }
243
243
  }
@@ -246,7 +246,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
246
246
  this.replicationJobs = newJobs;
247
247
  this.activeReplicationJob = activeJob;
248
248
 
249
- // Stop any orphaned jobs that no longer have sync rules.
249
+ // Stop any orphaned jobs that no longer have a replication stream.
250
250
  // Termination happens below
251
251
  for (let job of existingJobs.values()) {
252
252
  // Old - stop and clean up
@@ -254,11 +254,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
254
254
  await job.stop();
255
255
  } catch (e) {
256
256
  // This will be retried
257
- this.logger.warn('Failed to stop old replication job}', e);
257
+ job.storage.logger.warn('Failed to stop old replication job', e);
258
258
  }
259
259
  }
260
260
 
261
- // Sync rules stopped previously, including by a different process.
261
+ // Replication stream stopped previously, including by a different process.
262
262
  const stopped = await this.storage.getStoppedSyncRules();
263
263
  for (let syncRules of stopped) {
264
264
  if (this.clearingJobs.has(syncRules.id)) {
@@ -268,11 +268,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
268
268
 
269
269
  // We clear storage asynchronously.
270
270
  // It is important to be able to continue running the refresh loop, otherwise we cannot
271
- // retry locked sync rules, for example.
271
+ // retry locked replication stream, for example.
272
272
  const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
273
273
  const promise = this.terminateSyncRules(syncRuleStorage)
274
274
  .catch((e) => {
275
- this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
275
+ syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
276
276
  })
277
277
  .finally(() => {
278
278
  this.clearingJobs.delete(syncRules.id);
@@ -286,12 +286,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
286
286
  }
287
287
 
288
288
  protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
289
- this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
289
+ syncRuleStorage.logger.info(`Terminating replication stream...`);
290
290
  // This deletes postgres replication slots - should complete quickly.
291
291
  // It is safe to do before or after clearing the data in the storage.
292
292
  await this.cleanUp(syncRuleStorage);
293
293
  await syncRuleStorage.terminate({ signal: this.abortController?.signal, clearStorage: true });
294
- this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
294
+ syncRuleStorage.logger.info(`Successfully terminated replication stream`);
295
295
  }
296
296
 
297
297
  abstract testConnection(): Promise<ConnectionTestResult>;
@@ -119,7 +119,7 @@ export const reprocess = routeDefinition({
119
119
  const apiHandler = service_context.routerEngine.getAPI();
120
120
  const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
121
121
  if (next != null) {
122
- throw new Error(`Busy processing sync rules - cannot reprocess`);
122
+ throw new Error(`Busy processing sync config - cannot reprocess`);
123
123
  }
124
124
 
125
125
  const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
@@ -127,13 +127,17 @@ export const reprocess = routeDefinition({
127
127
  throw new errors.ServiceError({
128
128
  status: 422,
129
129
  code: ErrorCode.PSYNC_S4104,
130
- description: 'No active sync rules'
130
+ description: 'No active sync config'
131
131
  });
132
132
  }
133
133
 
134
+ // There are some differences between this and using asUpdateOptions():
135
+ // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
136
+ // 2. If the source does not set the storage version, this will update it do the current version.
137
+ // We can consider tweaking this behavior in the future.
134
138
  const new_rules = await activeBucketStorage.updateSyncRules(
135
139
  storage.updateSyncRulesFromYaml(active.sync_rules.config.content, {
136
- // These sync rules already passed validation. But if the rules are not valid anymore due
140
+ // This sync config already passed validation. But if the config is not valid anymore due
137
141
  // to a service change, we do want to report the error here.
138
142
  validate: true
139
143
  })
@@ -33,7 +33,7 @@ export const writeCheckpoint = routeDefinition({
33
33
  const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
34
34
  const cp = await bucketStorage?.getCheckpoint();
35
35
  if (cp == null) {
36
- throw new Error('No sync rules available');
36
+ throw new Error('No sync config available');
37
37
  }
38
38
  if (cp.lsn && cp.lsn >= head) {
39
39
  logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
@@ -77,7 +77,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
77
77
  new errors.ServiceError({
78
78
  status: 500,
79
79
  code: ErrorCode.PSYNC_S2302,
80
- description: 'No sync rules available'
80
+ description: 'No sync config available'
81
81
  })
82
82
  );
83
83
  responder.onComplete();
@@ -43,12 +43,12 @@ export const deploySyncRules = routeDefinition({
43
43
  const { storageEngine } = service_context;
44
44
 
45
45
  if (service_context.configuration.sync_rules.present) {
46
- // If sync rules are configured via the config, disable deploy via the API.
46
+ // If sync config is configured via the service config, disable deploy via the API.
47
47
  throw new errors.ServiceError({
48
48
  status: 422,
49
49
  code: ErrorCode.PSYNC_S4105,
50
- description: 'Sync rules API disabled',
51
- details: 'Use the management API to deploy sync rules'
50
+ description: 'Sync config API disabled',
51
+ details: 'Update sync config in the service configuration'
52
52
  });
53
53
  }
54
54
  const content = payload.params.content;
@@ -65,7 +65,7 @@ export const deploySyncRules = routeDefinition({
65
65
  throw new errors.ServiceError({
66
66
  status: 422,
67
67
  code: ErrorCode.PSYNC_R0001,
68
- description: 'Sync rules parsing failed',
68
+ description: 'Sync config parsing failed',
69
69
  details: e.message
70
70
  });
71
71
  }
@@ -115,7 +115,7 @@ export const currentSyncRules = routeDefinition({
115
115
  throw new errors.ServiceError({
116
116
  status: 422,
117
117
  code: ErrorCode.PSYNC_S4104,
118
- description: 'No active sync rules'
118
+ description: 'No active sync config'
119
119
  });
120
120
  }
121
121
 
@@ -162,13 +162,13 @@ export const reprocessSyncRules = routeDefinition({
162
162
  throw new errors.ServiceError({
163
163
  status: 422,
164
164
  code: ErrorCode.PSYNC_S4104,
165
- description: 'No active sync rules'
165
+ description: 'No active sync config'
166
166
  });
167
167
  }
168
168
 
169
169
  const new_rules = await activeBucketStorage.updateSyncRules(
170
170
  updateSyncRulesFromYaml(sync_rules.sync_rules.config.content, {
171
- // These sync rules already passed validation. But if the rules are not valid anymore due
171
+ // This sync config already passed validation. But if the rules are not valid anymore due
172
172
  // to a service change, we do want to report the error here.
173
173
  validate: true
174
174
  })
@@ -68,7 +68,7 @@ export const syncStreamed = routeDefinition({
68
68
  throw new errors.ServiceError({
69
69
  status: 500,
70
70
  code: ErrorCode.PSYNC_S2302,
71
- description: 'No sync rules available'
71
+ description: 'No sync config available'
72
72
  });
73
73
  }
74
74
 
@@ -121,7 +121,7 @@ export const syncStreamed = routeDefinition({
121
121
  });
122
122
 
123
123
  stream.on('end', () => {
124
- // Auth failure or switch to new sync rules
124
+ // Auth failure or switch to new sync config
125
125
  closeReason ??= 'service closing stream';
126
126
  });
127
127
 
@@ -35,13 +35,13 @@ export async function teardown(runnerConfig: utils.RunnerConfig) {
35
35
  }
36
36
 
37
37
  async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, moduleManager: modules.ModuleManager) {
38
- logger.info(`Terminating sync rules...`);
38
+ logger.info(`Terminating replication stream...`);
39
39
  const start = Date.now();
40
40
  const locks: storage.ReplicationLock[] = [];
41
41
  while (Date.now() - start < 120_000) {
42
42
  let retry = false;
43
43
  const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
44
- // Lock all the replicating sync rules
44
+ // Lock all the replicating replication streams
45
45
  for (const replicatingSyncRule of replicatingSyncRules) {
46
46
  const lock = await replicatingSyncRule.lock();
47
47
  locks.push(lock);
@@ -50,10 +50,10 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
50
50
  const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
51
51
  const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
52
52
  try {
53
- // Clean up any module specific configuration for the sync rules
53
+ // Clean up any module specific configuration for the replication stream
54
54
  await moduleManager.tearDown({ syncRules: combinedSyncRules });
55
55
 
56
- // Mark the sync rules as terminated
56
+ // Mark the replication stream as terminated
57
57
  for (let syncRules of combinedSyncRules) {
58
58
  const syncRulesStorage = storageFactory.getInstance(syncRules);
59
59
  // The storage will be dropped at the end of the teardown, so we don't need to clear it here
@@ -2,36 +2,36 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
2
2
 
3
3
  export enum SyncRuleState {
4
4
  /**
5
- * New sync rules - needs to be processed (initial replication).
5
+ * New replication stream - needs to be processed (initial replication).
6
6
  *
7
- * While multiple sets of sync rules _can_ be in PROCESSING,
7
+ * While multiple replication streams _can_ be in PROCESSING,
8
8
  * it's generally pointless, so we only keep one in that state.
9
9
  */
10
10
  PROCESSING = 'PROCESSING',
11
11
 
12
12
  /**
13
- * Sync rule processing is done, and can be used for sync.
13
+ * Intial processing is done, and can be used for sync.
14
14
  *
15
- * Only one set of sync rules should be in ACTIVE or ERRORED state.
15
+ * Only one replication stream should be in ACTIVE or ERRORED state.
16
16
  */
17
17
  ACTIVE = 'ACTIVE',
18
18
  /**
19
- * This state is used when the sync rules has been replaced,
19
+ * This state is used when the replication stream has been replaced,
20
20
  * and replication is or should be stopped.
21
21
  */
22
22
  STOP = 'STOP',
23
23
  /**
24
- * After sync rules have been stopped, the data needs to be
24
+ * After replication stream has been stopped, the data needs to be
25
25
  * deleted. Once deleted, the state is TERMINATED.
26
26
  */
27
27
  TERMINATED = 'TERMINATED',
28
28
 
29
29
  /**
30
- * Sync rules has run into a permanent replication error. It
31
- * is still the "active" sync rules for syncing to users,
30
+ * Replication stream has run into a permanent replication error. It
31
+ * is still the "active" replication stram for syncing to users,
32
32
  * but should not replicate anymore.
33
33
  *
34
- * It will transition to STOP when a new sync rules is activated.
34
+ * It will transition to STOP when a new replication stream is activated.
35
35
  */
36
36
  ERRORED = 'ERRORED'
37
37
  }
@@ -17,17 +17,17 @@ import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
17
17
  /**
18
18
  * Represents a configured storage provider.
19
19
  *
20
- * The provider can handle multiple copies of sync rules concurrently, each with their own storage.
21
- * This is to handle replication of a new version of sync rules, while the old version is still active.
20
+ * The provider can handle multiple replication streams concurrently, each with their own storage.
21
+ * This is to handle replication of a new version of sync config, while the old replication stream is still active.
22
22
  *
23
- * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
23
+ * Storage APIs for a specific replication stream are provided by the `SyncRulesBucketStorage` instances.
24
24
  */
25
25
  export abstract class BucketStorageFactory
26
26
  extends BaseObserver<BucketStorageFactoryListener>
27
27
  implements AsyncDisposable
28
28
  {
29
29
  /**
30
- * Update sync rules from configuration, if changed.
30
+ * Update sync config from configuration, if changed.
31
31
  */
32
32
  async configureSyncRules(
33
33
  options: UpdateSyncRulesOptions
@@ -36,42 +36,42 @@ export abstract class BucketStorageFactory
36
36
  const active = await this.getActiveSyncRulesContent();
37
37
 
38
38
  if (next?.sync_rules_content == options.config.yaml) {
39
- logger.info('Sync rules from configuration unchanged');
39
+ logger.info('Sync config unchanged');
40
40
  return { updated: false };
41
41
  } else if (next == null && active?.sync_rules_content == options.config.yaml) {
42
- logger.info('Sync rules from configuration unchanged');
42
+ logger.info('Sync config unchanged');
43
43
  return { updated: false };
44
44
  } else {
45
- logger.info('Sync rules updated from configuration');
45
+ logger.info('Sync config updated');
46
46
  const persisted_sync_rules = await this.updateSyncRules(options);
47
47
  return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
48
48
  }
49
49
  }
50
50
 
51
51
  /**
52
- * Get a storage instance to query sync data for specific sync rules.
52
+ * Get a storage instance to query sync data for specific sync config.
53
53
  */
54
54
  abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
55
55
 
56
56
  /**
57
- * Deploy new sync rules.
57
+ * Deploy new sync config.
58
58
  */
59
59
  abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
60
60
 
61
61
  /**
62
62
  * Indicate that a slot was removed, and we should re-sync by creating
63
- * a new sync rules instance.
63
+ * a new replication stream.
64
64
  *
65
65
  * This is roughly the same as deploying a new version of the current sync
66
- * rules, but also accounts for cases where the current sync rules are not
67
- * the latest ones.
66
+ * config, but also accounts for cases where the current sync config is not
67
+ * the latest one.
68
68
  *
69
69
  * Replication should be restarted after this.
70
70
  */
71
71
  abstract restartReplication(sync_rules_group_id: number): Promise<void>;
72
72
 
73
73
  /**
74
- * Get the sync rules used for querying.
74
+ * Get the sync config used for querying.
75
75
  */
76
76
  async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
77
77
  const content = await this.getActiveSyncRulesContent();
@@ -79,12 +79,12 @@ export abstract class BucketStorageFactory
79
79
  }
80
80
 
81
81
  /**
82
- * Get the sync rules used for querying.
82
+ * Get the sync config used for querying.
83
83
  */
84
84
  abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
85
85
 
86
86
  /**
87
- * Get the sync rules that will be active next once done with initial replicatino.
87
+ * Get the sync config that will be active next once done with initial replicatino.
88
88
  */
89
89
  async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
90
90
  const content = await this.getNextSyncRulesContent();
@@ -92,17 +92,17 @@ export abstract class BucketStorageFactory
92
92
  }
93
93
 
94
94
  /**
95
- * Get the sync rules that will be active next once done with initial replicatino.
95
+ * Get the sync config that will be active next once done with initial replicatino.
96
96
  */
97
97
  abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
98
98
 
99
99
  /**
100
- * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
100
+ * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
101
101
  */
102
102
  abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
103
103
 
104
104
  /**
105
- * Get all sync rules stopped but not terminated yet.
105
+ * Get all sync config stopped but not terminated yet.
106
106
  */
107
107
  abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
108
108
 
@@ -112,12 +112,12 @@ export abstract class BucketStorageFactory
112
112
  abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
113
113
 
114
114
  /**
115
- * Get storage size of active sync rules.
115
+ * Get storage size of active replication stream.
116
116
  */
117
117
  abstract getStorageMetrics(): Promise<StorageMetrics>;
118
118
 
119
119
  /**
120
- * Get the unique identifier for this instance of Powersync
120
+ * Get the unique identifier for this instance of Powersync.
121
121
  */
122
122
  abstract getPowerSyncInstanceId(): Promise<string>;
123
123
 
@@ -161,6 +161,12 @@ export interface UpdateSyncRulesOptions {
161
161
  * compiler.
162
162
  */
163
163
  plan: SerializedSyncPlan | null;
164
+
165
+ /**
166
+ * Parsed sync config, primarily to generate a definition mapping.
167
+ * Not persisted, and the defaultSchema used for parsing is not relevant.
168
+ */
169
+ parsed: SyncConfigWithErrors;
164
170
  };
165
171
  lock?: boolean;
166
172
  storageVersion?: number;
@@ -198,10 +204,11 @@ export function updateSyncRulesFromYaml(
198
204
  }
199
205
 
200
206
  export function updateSyncRulesFromConfig(
201
- { config, errors }: SyncConfigWithErrors,
207
+ parsed: SyncConfigWithErrors,
202
208
  options?: Omit<UpdateSyncRulesOptions, 'config'>
203
209
  ): UpdateSyncRulesOptions {
204
210
  let plan: SerializedSyncPlan | null = null;
211
+ const { config, errors } = parsed;
205
212
  if (config instanceof PrecompiledSyncConfig) {
206
213
  const eventDescriptors: Record<string, string[]> = {};
207
214
  for (const event of config.eventDescriptors) {
@@ -216,7 +223,7 @@ export function updateSyncRulesFromConfig(
216
223
  };
217
224
  }
218
225
 
219
- return { config: { yaml: config.content, plan }, ...options };
226
+ return { config: { yaml: config.content, plan, parsed }, ...options };
220
227
  }
221
228
 
222
229
  export interface GetIntanceOptions {
@@ -1,4 +1,4 @@
1
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
1
+ import { logger as defaultLogger, ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
2
2
  import {
3
3
  CompatibilityContext,
4
4
  CompatibilityOption,
@@ -29,7 +29,7 @@ export interface PersistedSyncRulesContentData {
29
29
  readonly compiled_plan: SerializedSyncPlan | null;
30
30
  readonly slot_name: string;
31
31
  /**
32
- * True if this is the "active" copy of the sync rules.
32
+ * True if this is the "active" copy of the sync config.
33
33
  */
34
34
  readonly active: boolean;
35
35
  readonly storageVersion: number;
@@ -49,6 +49,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
49
49
  readonly slot_name!: string;
50
50
  readonly active!: boolean;
51
51
  readonly storageVersion!: number;
52
+ readonly logger: Logger;
52
53
 
53
54
  readonly last_checkpoint_lsn!: string | null;
54
55
 
@@ -61,6 +62,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
61
62
 
62
63
  constructor(data: PersistedSyncRulesContentData) {
63
64
  Object.assign(this, data);
65
+ this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` });
64
66
  }
65
67
 
66
68
  /**
@@ -73,7 +75,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
73
75
  if (storageConfig == null) {
74
76
  throw new ServiceError(
75
77
  ErrorCode.PSYNC_S1005,
76
- `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
78
+ `Unsupported storage version ${this.storageVersion} for replication stream ${this.id}`
77
79
  );
78
80
  }
79
81
  return storageConfig;
@@ -103,6 +105,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
103
105
  sourceText: this.sync_rules_content
104
106
  });
105
107
 
108
+ // Note: If the original content did not define a storage version, this will still set the storage version.
109
+ // This means asUpdateOptions will not change the storage version, even if the default changes.
110
+ precompiled.storageVersion = this.storageVersion;
111
+
106
112
  const errors: YamlError[] = [];
107
113
  if (this.compiled_plan.errors) {
108
114
  for (const error of this.compiled_plan.errors) {
@@ -144,8 +150,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
144
150
  }
145
151
 
146
152
  asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
153
+ // defaultSchema is not relevant for the parsed version here
154
+ const parsed = this.parsed({ defaultSchema: 'not_applicable' });
147
155
  return {
148
- config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
156
+ config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules },
149
157
  ...options
150
158
  };
151
159
  }
@@ -28,7 +28,7 @@ export class SourceTable implements SourceEntityDescriptor {
28
28
  static readonly DEFAULT_TAG = DEFAULT_TAG;
29
29
 
30
30
  /**
31
- * True if the table is used in sync rules for data queries.
31
+ * True if the table is used in sync config for data queries.
32
32
  *
33
33
  * This value is resolved externally, and cached here.
34
34
  *
@@ -37,7 +37,7 @@ export class SourceTable implements SourceEntityDescriptor {
37
37
  public syncData = true;
38
38
 
39
39
  /**
40
- * True if the table is used in sync rules for data queries.
40
+ * True if the table is used in sync config for data queries.
41
41
  *
42
42
  * This value is resolved externally, and cached here.
43
43
  *
@@ -46,7 +46,7 @@ export class SourceTable implements SourceEntityDescriptor {
46
46
  public syncParameters = true;
47
47
 
48
48
  /**
49
- * True if the table is used in sync rules for events.
49
+ * True if the table is used in sync config for events.
50
50
  *
51
51
  * This value is resolved externally, and cached here.
52
52
  *
@@ -45,7 +45,7 @@ export const STORAGE_VERSION_3 = 3;
45
45
  export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
46
46
 
47
47
  /**
48
- * Default storage version for newly persisted sync rules.
48
+ * Default storage version for newly persisted replication streams.
49
49
  */
50
50
  export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
51
51