@powersync/service-core 1.23.0 → 1.23.1

package/src/replication/AbstractReplicator.ts

@@ -1,6 +1,7 @@
 import { container, ErrorCode, logger } from '@powersync/lib-services-framework';
 import { ReplicationMetric } from '@powersync/service-types';
 import { hrtime } from 'node:process';
+import { setTimeout as sleep } from 'node:timers/promises';
 import winston from 'winston';
 import { MetricsEngine } from '../metrics/MetricsEngine.js';
 import * as storage from '../storage/storage-index.js';
@@ -13,6 +14,12 @@ import { ConnectionTestResult } from './ReplicationModule.js';
 // 1 minute
 const PING_INTERVAL = 1_000_000_000n * 60n;
 
+// In the initial startup period, we use a short refresh interval. This helps to take over replication quickly in the case
+// of rolling deploys. After the initial period, we switch to a longer refresh interval to reduce load.
+const FAST_REFRESH_INTERVAL_MS = 500;
+const FAST_REFRESH_TIMEOUT_MS = 120_000;
+const REFRESH_INTERVAL_MS = 5_000;
+
 export interface CreateJobOptions {
   lock: storage.ReplicationLock;
   storage: storage.SyncRulesBucketStorage;
@@ -36,7 +43,7 @@ export interface AbstractReplicatorOptions {
  */
 export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
   protected logger: winston.Logger;
-  private lockAlerted: boolean = false;
+  private lastReplicationStreamInfoLogs = new Map<string, string>();
   /**
    *  Map of replication jobs by replication stream job id. Usually there is only one running job, but there could be two
    *  when transitioning to a new replication stream.
@@ -131,17 +138,20 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
   }
 
   private async runLoop() {
-    const syncRules = await this.syncRuleProvider.get();
+    const loadedSyncConfig = await this.syncRuleProvider.get();
 
     let configuredLock: storage.ReplicationLock | undefined = undefined;
-    if (syncRules != null) {
+    if (loadedSyncConfig != null) {
       this.logger.info('Loaded sync config');
       try {
         // Configure new sync config, if they have changed.
         // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+        // This upfront lock is mostly superseded by the replicationStreamLoadedSyncConfigMatch() check. However, that doesn't cover old
+        // versions before that check was added, so we keep the lock for now - for where te service version and sync config is updated at
+        // the same time.
 
         const { lock } = await this.storage.configureSyncRules(
-          storage.updateSyncRulesFromYaml(syncRules, { lock: true, validate: this.syncRuleProvider.exitOnError })
+          storage.updateSyncRulesFromYaml(loadedSyncConfig, { lock: true, validate: this.syncRuleProvider.exitOnError })
         );
         if (lock) {
           configuredLock = lock;
@@ -155,10 +165,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     } else {
       this.logger.info('No sync streams or rules configured - configure via API');
     }
+    let useFastRefresh = true;
+    const fastRefreshDeadline = Date.now() + FAST_REFRESH_TIMEOUT_MS;
     while (!this.stopped) {
       await container.probes.touch();
       try {
-        await this.refresh({ configured_lock: configuredLock });
+        const refreshResult = await this.refresh({ configuredLock, loadedSyncConfig });
+        if (refreshResult.replicationJobStarted || Date.now() >= fastRefreshDeadline) {
+          useFastRefresh = false;
+        }
         // The lock is only valid on the first refresh.
         configuredLock = undefined;
 
@@ -177,24 +192,40 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
       } catch (e) {
         this.logger.error('Failed to refresh replication jobs', e);
       }
-      await new Promise((resolve) => setTimeout(resolve, 5000));
+      if (Date.now() >= fastRefreshDeadline) {
+        useFastRefresh = false;
+      }
+      await sleep(useFastRefresh ? FAST_REFRESH_INTERVAL_MS : REFRESH_INTERVAL_MS);
     }
   }
 
-  private async refresh(options?: { configured_lock?: storage.ReplicationLock }) {
+  private async refresh(options?: {
+    configuredLock?: storage.ReplicationLock;
+    loadedSyncConfig?: string;
+  }): Promise<{ replicationJobStarted: boolean }> {
     if (this.stopped) {
-      return;
+      return { replicationJobStarted: false };
     }
 
-    let configuredLock = options?.configured_lock;
+    let configuredLock = options?.configuredLock;
+    let replicationJobStarted = false;
 
     const existingJobs = new Map<string, T>(this.replicationJobs.entries());
     const replicatingStreams = await this.storage.getReplicatingReplicationStreams();
     const newJobs = new Map<string, T>();
+    const streamsToStart: storage.PersistedReplicationStream[] = [];
     let activeJob: T | undefined = undefined;
     for (let replicationStream of replicatingStreams) {
       const jobId = replicationStream.replicationJobId;
       const existingJob = existingJobs.get(jobId);
+      const syncConfigMismatchMessage = 'Ignoring replication stream for sync config not loaded by this process';
+      if (!this.shouldHandleReplicationStream(replicationStream, options?.loadedSyncConfig)) {
+        this.logReplicationStreamInfoOnce(replicationStream, 'sync-config-mismatch', () => {
+          replicationStream.logger.info(syncConfigMismatchMessage);
+        });
+        continue;
+      }
+      this.clearReplicationStreamInfoLog(replicationStream, 'sync-config-mismatch');
       if (replicationStream.state == storage.SyncRuleState.ACTIVE && activeJob == null) {
         activeJob = existingJob;
       }
@@ -208,47 +239,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
         existingJobs.delete(jobId);
       } else {
         // New sync config was found (or resume after restart)
-        try {
-          let lock: storage.ReplicationLock;
-          if (configuredLock?.sync_rules_id == replicationStream.replicationStreamId) {
-            lock = configuredLock;
-          } else {
-            lock = await replicationStream.lock();
-          }
-          const syncRuleStorage = this.storage.getInstance(replicationStream);
-          const newJob = this.createJob({
-            lock: lock,
-            storage: syncRuleStorage
-          });
-
-          newJobs.set(jobId, newJob);
-          newJob.start();
-          if (replicationStream.state == storage.SyncRuleState.ACTIVE) {
-            activeJob = newJob;
-          }
-          this.lockAlerted = false;
-        } catch (e) {
-          if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
-            if (!this.lockAlerted) {
-              replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
-              this.lockAlerted = true;
-            }
-          } else {
-            // Could be a sync config parse error,
-            // for example from stricter validation that was added.
-            // This will be retried every couple of seconds.
-            // When new (valid) sync config is deployed and processed, this one be disabled.
-            replicationStream.logger.error('Failed to start replication for new sync config', e);
-          }
-        }
+        streamsToStart.push(replicationStream);
       }
     }
 
-    this.replicationJobs = newJobs;
-    this.activeReplicationJob = activeJob;
-
-    // Stop any orphaned jobs that no longer have a replication stream.
-    // Termination happens below
+    // Stop any orphaned jobs that no longer have a replication stream before starting replacements.
+    // Termination happens below.
     for (let job of existingJobs.values()) {
       // Old - stop and clean up
       try {
@@ -259,6 +255,46 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
       }
     }
 
+    for (let replicationStream of streamsToStart) {
+      const jobId = replicationStream.replicationJobId;
+      try {
+        let lock: storage.ReplicationLock;
+        if (configuredLock?.sync_rules_id == replicationStream.replicationStreamId) {
+          lock = configuredLock;
+        } else {
+          lock = await replicationStream.lock();
+        }
+        const syncRuleStorage = this.storage.getInstance(replicationStream);
+        const newJob = this.createJob({
+          lock: lock,
+          storage: syncRuleStorage
+        });
+
+        newJobs.set(jobId, newJob);
+        newJob.start();
+        replicationJobStarted = true;
+        if (replicationStream.state == storage.SyncRuleState.ACTIVE) {
+          activeJob = newJob;
+        }
+        this.lastReplicationStreamInfoLogs.delete(replicationStream.replicationStreamName);
+      } catch (e) {
+        if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
+          this.logReplicationStreamInfoOnce(replicationStream, 'replication-stream-locked', () => {
+            replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
+          });
+        } else {
+          // Could be a sync config parse error,
+          // for example from stricter validation that was added.
+          // This will be retried every couple of seconds.
+          // When new (valid) sync config is deployed and processed, this one be disabled.
+          replicationStream.logger.error('Failed to start replication for new sync config', e);
+        }
+      }
+    }
+
+    this.replicationJobs = newJobs;
+    this.activeReplicationJob = activeJob;
+
     // Replication stream stopped previously, including by a different process.
     const stopped = await this.storage.getStoppedReplicationStreams();
     for (let replicationStream of stopped) {
@@ -280,6 +316,49 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
         });
       this.clearingJobs.set(replicationStream.replicationStreamId, promise);
     }
+
+    return { replicationJobStarted };
+  }
+
+  private logReplicationStreamInfoOnce(
+    replicationStream: storage.PersistedReplicationStream,
+    category: string,
+    log: () => void
+  ) {
+    if (this.lastReplicationStreamInfoLogs.get(replicationStream.replicationStreamName) == category) {
+      return;
+    }
+
+    log();
+    this.lastReplicationStreamInfoLogs.set(replicationStream.replicationStreamName, category);
+  }
+
+  private clearReplicationStreamInfoLog(replicationStream: storage.PersistedReplicationStream, category: string) {
+    if (this.lastReplicationStreamInfoLogs.get(replicationStream.replicationStreamName) == category) {
+      this.lastReplicationStreamInfoLogs.delete(replicationStream.replicationStreamName);
+    }
+  }
+
+  /**
+   * When we load sync config from the filesystem/config source, we ignore any config loaded by other processes.
+   * The idea is that if a different process loads updated config, that process should process it, not this one.
+   *
+   * This specifically helps for cases with rolling deploys.
+   *
+   * This does not apply when sync config is loaded from the database/API, instead of in the config.
+   */
+  private shouldHandleReplicationStream(
+    replicationStream: storage.PersistedReplicationStream,
+    loadedSyncRules: string | undefined
+  ) {
+    if (loadedSyncRules == null) {
+      return true;
+    }
+
+    const processingConfig = replicationStream.syncConfigContent.find(
+      (syncConfig) => syncConfig.syncConfigState == storage.SyncRuleState.PROCESSING
+    );
+    return processingConfig == null || processingConfig.sync_rules_content == loadedSyncRules;
   }
 
   protected createJobId(syncRuleId: number) {

package/src/routes/endpoints/admin.ts

@@ -241,7 +241,8 @@ export const validate = routeDefinition({
       replicationStreamName: '',
       storageVersion: storage.LEGACY_STORAGE_VERSION,
       sync_rules_content: content,
-      compiled_plan: null
+      compiled_plan: null,
+      syncConfigState: storage.SyncRuleState.PROCESSING
     });
 
     const connectionStatus = await apiHandler.getConnectionStatus();

package/src/storage/PersistedSyncConfigContent.ts

@@ -17,6 +17,7 @@ import {
 } from '@powersync/service-sync-rules';
 import * as sqlite from 'node:sqlite';
 import { Logger } from 'winston';
+import { SyncRuleState } from './BucketStorage.js';
 import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
 import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
@@ -91,6 +92,7 @@ export abstract class PersistedSyncConfigContent implements PersistedSyncConfigC
   readonly storageVersion: number;
   readonly logger: Logger;
   readonly syncConfigId: PersistedSyncConfigId | null;
+  readonly syncConfigState: SyncRuleState;
 
   constructor(data: PersistedSyncConfigContentData) {
     this.replicationStreamId = data.replicationStreamId;
@@ -99,6 +101,7 @@ export abstract class PersistedSyncConfigContent implements PersistedSyncConfigC
     this.replicationStreamName = data.replicationStreamName;
     this.storageVersion = data.storageVersion;
     this.syncConfigId = data.syncConfigId ?? null;
+    this.syncConfigState = data.syncConfigState;
     this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
   }
 
@@ -181,6 +184,7 @@ export interface PersistedSyncConfigContentData {
   readonly storageVersion: number;
 
   readonly syncConfigId?: PersistedSyncConfigId | null;
+  readonly syncConfigState: SyncRuleState;
 }
 export type PersistedSyncConfigId = string;
 export interface ParseSyncConfigOptions {

package/test/src/diagnostics.test.ts

@@ -18,6 +18,17 @@ function makeSyncRulesContent(overrides?: {
   status?: storage.PersistedSyncConfigStatus;
 }): storage.PersistedSyncConfigContent {
   // We don't implement the entire interface correctly here - just enough to test the diagnostics logic.
+  const status = overrides?.status ?? {
+    id: '1',
+    replicationStreamId: 1,
+    state: storage.SyncRuleState.ACTIVE,
+    last_checkpoint_lsn: 'some_lsn',
+    last_fatal_error: null,
+    last_fatal_error_ts: null,
+    last_keepalive_ts: new Date(),
+    last_checkpoint_ts: new Date()
+  };
+
   return {
     replicationStreamId: 1,
     syncConfigId: null,
@@ -25,6 +36,7 @@ function makeSyncRulesContent(overrides?: {
     sync_rules_content: MINIMAL_SYNC_RULES,
     compiled_plan: null,
     storageVersion: 1,
+    syncConfigState: status.state,
     parsed(options?: any) {
       const syncRules = SqlSyncRules.fromYaml(MINIMAL_SYNC_RULES, {
         ...options,
@@ -38,18 +50,7 @@ function makeSyncRulesContent(overrides?: {
     asUpdateOptions: null as any,
     getStorageConfig: null as any,
     async getSyncConfigStatus() {
-      return (
-        overrides?.status ?? {
-          id: '1',
-          replicationStreamId: 1,
-          state: storage.SyncRuleState.ACTIVE,
-          last_checkpoint_lsn: 'some_lsn',
-          last_fatal_error: null,
-          last_fatal_error_ts: null,
-          last_keepalive_ts: new Date(),
-          last_checkpoint_ts: new Date()
-        }
-      );
+      return status;
     }
   } as storage.PersistedSyncConfigContent;
 }

package/test/src/routes/admin.test.ts

@@ -41,6 +41,16 @@ describe('admin routes', () => {
     const state = active ? storage.SyncRuleState.ACTIVE : storage.SyncRuleState.PROCESSING;
     const lastKeepaliveTs = new Date('2026-01-01T00:00:00.000Z');
     const lastCheckpointTs = new Date('2026-01-01T00:00:00.000Z');
+    const syncConfigStatus = {
+      id: syncConfigId,
+      replicationStreamId: id,
+      state,
+      last_checkpoint_lsn: null,
+      last_fatal_error: null,
+      last_fatal_error_ts: null,
+      last_keepalive_ts: lastKeepaliveTs,
+      last_checkpoint_ts: lastCheckpointTs
+    };
     const content = {
       replicationStreamId: id,
       syncConfigId,
@@ -55,6 +65,7 @@ bucket_definitions:
 `,
       compiled_plan: null,
       storageVersion: storage.LEGACY_STORAGE_VERSION,
+      syncConfigState: syncConfigStatus.state,
       parsed(options?: any) {
         const syncRules = SqlSyncRules.fromYaml(content.sync_rules_content, {
           ...options,
@@ -67,16 +78,7 @@ bucket_definitions:
       asUpdateOptions: vi.fn(),
       getStorageConfig: vi.fn(),
       async getSyncConfigStatus() {
-        return {
-          id: syncConfigId,
-          replicationStreamId: id,
-          state,
-          last_checkpoint_lsn: null,
-          last_fatal_error: null,
-          last_fatal_error_ts: null,
-          last_keepalive_ts: lastKeepaliveTs,
-          last_checkpoint_ts: lastCheckpointTs
-        };
+        return syncConfigStatus;
       }
     };
     return content as unknown as storage.PersistedSyncConfigContent;