@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/replication/AbstractReplicationJob.ts

@@ -0,0 +1,79 @@
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+import { container, logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+
+export interface AbstractReplicationJobOptions {
+  id: string;
+  storage: storage.SyncRulesBucketStorage;
+  lock: storage.ReplicationLock;
+  rateLimiter: ErrorRateLimiter;
+}
+
+export abstract class AbstractReplicationJob {
+  protected logger: winston.Logger;
+  protected abortController = new AbortController();
+  protected isReplicatingPromise: Promise<void> | null = null;
+
+  protected constructor(protected options: AbstractReplicationJobOptions) {
+    this.logger = logger.child({ name: `ReplicationJob: ${options.id}` });
+  }
+
+  /**
+   *  Copy the initial data set from the data source if required and then keep it in sync.
+   */
+  abstract replicate(): Promise<void>;
+
+  /**
+   *  Ensure the connection to the data source remains active
+   */
+  abstract keepAlive(): Promise<void>;
+
+  /**
+   *  Start the replication process
+   */
+  public async start(): Promise<void> {
+    this.isReplicatingPromise = this.replicate()
+      .catch((ex) => {
+        container.reporter.captureException(ex, {
+          metadata: {
+            replicator: this.id
+          }
+        });
+        this.logger.error(`Replication failed.`, ex);
+      })
+      .finally(async () => {
+        this.abortController.abort();
+        await this.options.lock.release();
+      });
+  }
+
+  /**
+   *  Safely stop the replication process
+   */
+  public async stop(): Promise<void> {
+    this.logger.info(`Stopping ${this.id} replication job for sync rule iteration: ${this.storage.group_id}`);
+    this.abortController.abort();
+    await this.isReplicatingPromise;
+  }
+
+  public get id() {
+    return this.options.id;
+  }
+
+  public get storage() {
+    return this.options.storage;
+  }
+
+  protected get lock() {
+    return this.options.lock;
+  }
+
+  protected get rateLimiter() {
+    return this.options.rateLimiter;
+  }
+
+  public get isStopped(): boolean {
+    return this.abortController.signal.aborted;
+  }
+}

package/src/replication/AbstractReplicator.ts

@@ -0,0 +1,227 @@
+import { hrtime } from 'node:process';
+import * as storage from '../storage/storage-index.js';
+import { container, logger } from '@powersync/lib-services-framework';
+import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
+import winston from 'winston';
+import { AbstractReplicationJob } from './AbstractReplicationJob.js';
+import { StorageEngine } from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+
+// 5 minutes
+const PING_INTERVAL = 1_000_000_000n * 300n;
+
+export interface CreateJobOptions {
+  lock: storage.ReplicationLock;
+  storage: storage.SyncRulesBucketStorage;
+}
+
+export interface AbstractReplicatorOptions {
+  id: string;
+  storageEngine: StorageEngine;
+  syncRuleProvider: SyncRulesProvider;
+  /**
+   * This limits the effect of retries when there is a persistent issue.
+   */
+  rateLimiter: ErrorRateLimiter;
+}
+
+/**
+ *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
+ *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
+ *   It also handles any changes to the sync rules.
+ */
+export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
+  protected logger: winston.Logger;
+  /**
+   *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
+   *  transitioning to a new set of sync rules.
+   *  @private
+   */
+  private replicationJobs = new Map<number, T>();
+  private stopped = false;
+
+  // First ping is only after 5 minutes, not when starting
+  private lastPing = hrtime.bigint();
+
+  protected constructor(private options: AbstractReplicatorOptions) {
+    this.logger = logger.child({ name: `Replicator:${options.id}` });
+  }
+
+  abstract createJob(options: CreateJobOptions): T;
+
+  /**
+   *  Clean up any configuration or state for the specified sync rule on the datasource.
+   *  Should be a no-op if the configuration has already been cleared
+   */
+  abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
+
+  public get id() {
+    return this.options.id;
+  }
+
+  protected get storage() {
+    return this.options.storageEngine.activeBucketStorage;
+  }
+
+  protected get syncRuleProvider() {
+    return this.options.syncRuleProvider;
+  }
+
+  protected get rateLimiter() {
+    return this.options.rateLimiter;
+  }
+
+  public async start(): Promise<void> {
+    this.runLoop().catch((e) => {
+      this.logger.error('Data source fatal replication error', e);
+      container.reporter.captureException(e);
+      setTimeout(() => {
+        process.exit(1);
+      }, 1000);
+    });
+  }
+
+  public async stop(): Promise<void> {
+    this.stopped = true;
+    let promises: Promise<void>[] = [];
+    for (const job of this.replicationJobs.values()) {
+      promises.push(job.stop());
+    }
+    await Promise.all(promises);
+  }
+
+  private async runLoop() {
+    const syncRules = await this.syncRuleProvider.get();
+    let configuredLock: storage.ReplicationLock | undefined = undefined;
+    if (syncRules != null) {
+      this.logger.info('Loaded sync rules');
+      try {
+        // Configure new sync rules, if they have changed.
+        // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+        const { lock } = await this.storage.configureSyncRules(syncRules, {
+          lock: true
+        });
+        if (lock) {
+          configuredLock = lock;
+        }
+      } catch (e) {
+        // Log, but continue with previous sync rules
+        this.logger.error(`Failed to update sync rules from configuration`, e);
+      }
+    } else {
+      this.logger.info('No sync rules configured - configure via API');
+    }
+    while (!this.stopped) {
+      await container.probes.touch();
+      try {
+        await this.refresh({ configured_lock: configuredLock });
+        // The lock is only valid on the first refresh.
+        configuredLock = undefined;
+
+        // Ensure that the replication jobs' connections are kept alive.
+        // We don't ping while in error retry back-off, to avoid having too failures.
+        if (this.rateLimiter.mayPing()) {
+          const now = hrtime.bigint();
+          if (now - this.lastPing >= PING_INTERVAL) {
+            for (const activeJob of this.replicationJobs.values()) {
+              await activeJob.keepAlive();
+            }
+
+            this.lastPing = now;
+          }
+        }
+      } catch (e) {
+        this.logger.error('Failed to refresh replication jobs', e);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+    }
+  }
+
+  private async refresh(options?: { configured_lock?: storage.ReplicationLock }) {
+    if (this.stopped) {
+      return;
+    }
+
+    let configuredLock = options?.configured_lock;
+
+    const existingJobs = new Map<number, T>(this.replicationJobs.entries());
+    const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
+    const newJobs = new Map<number, T>();
+    for (let syncRules of replicatingSyncRules) {
+      const existingJob = existingJobs.get(syncRules.id);
+      if (existingJob && !existingJob.isStopped) {
+        // No change
+        existingJobs.delete(syncRules.id);
+        newJobs.set(syncRules.id, existingJob);
+      } else if (existingJob && existingJob.isStopped) {
+        // Stopped (e.g. fatal error).
+        // Remove from the list. Next refresh call will restart the job.
+        existingJobs.delete(syncRules.id);
+      } else {
+        // New sync rules were found (or resume after restart)
+        try {
+          let lock: storage.ReplicationLock;
+          if (configuredLock?.sync_rules_id == syncRules.id) {
+            lock = configuredLock;
+          } else {
+            lock = await syncRules.lock();
+          }
+          const storage = this.storage.getInstance(syncRules);
+          const newJob = this.createJob({
+            lock: lock,
+            storage: storage
+          });
+
+          newJobs.set(syncRules.id, newJob);
+          newJob.start();
+        } catch (e) {
+          // Could be a sync rules parse error,
+          // for example from stricter validation that was added.
+          // This will be retried every couple of seconds.
+          // When new (valid) sync rules are deployed and processed, this one be disabled.
+          this.logger.error('Failed to start replication for new sync rules', e);
+        }
+      }
+    }
+
+    this.replicationJobs = newJobs;
+
+    // Terminate any orphaned jobs that no longer have sync rules
+    for (let job of existingJobs.values()) {
+      // Old - stop and clean up
+      try {
+        await job.stop();
+        await this.terminateSyncRules(job.storage);
+      } catch (e) {
+        // This will be retried
+        this.logger.warn('Failed to terminate old replication job}', e);
+      }
+    }
+
+    // Sync rules stopped previously or by a different process.
+    const stopped = await this.storage.getStoppedSyncRules();
+    for (let syncRules of stopped) {
+      try {
+        const syncRuleStorage = this.storage.getInstance(syncRules);
+        await this.terminateSyncRules(syncRuleStorage);
+      } catch (e) {
+        this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
+      }
+    }
+  }
+
+  protected createJobId(syncRuleId: number) {
+    return `${this.id}-${syncRuleId}`;
+  }
+
+  protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
+    this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
+    try {
+      await this.cleanUp(syncRuleStorage);
+      await syncRuleStorage.terminate();
+      this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
+    } catch (e) {
+      this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
+    }
+  }
+}

package/src/replication/ErrorRateLimiter.ts

@@ -1,50 +1,6 @@
-import { setTimeout } from 'timers/promises';
-
 export interface ErrorRateLimiter {
   waitUntilAllowed(options?: { signal?: AbortSignal }): Promise<void>;
   reportError(e: any): void;
 
   mayPing(): boolean;
 }
-
-export class DefaultErrorRateLimiter implements ErrorRateLimiter {
-  nextAllowed: number = Date.now();
-
-  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
-    const delay = Math.max(0, this.nextAllowed - Date.now());
-    // Minimum delay between connections, even without errors
-    this.setDelay(500);
-    await setTimeout(delay, undefined, { signal: options?.signal });
-  }
-
-  mayPing(): boolean {
-    return Date.now() >= this.nextAllowed;
-  }
-
-  reportError(e: any): void {
-    const message = (e.message as string) ?? '';
-    if (message.includes('password authentication failed')) {
-      // Wait 15 minutes, to avoid triggering Supabase's fail2ban
-      this.setDelay(900_000);
-    } else if (message.includes('ENOTFOUND')) {
-      // DNS lookup issue - incorrect URI or deleted instance
-      this.setDelay(120_000);
-    } else if (message.includes('ECONNREFUSED')) {
-      // Could be fail2ban or similar
-      this.setDelay(120_000);
-    } else if (
-      message.includes('Unable to do postgres query on ended pool') ||
-      message.includes('Postgres unexpectedly closed connection')
-    ) {
-      // Connection timed out - ignore / immediately retry
-      // We don't explicitly set the delay to 0, since there could have been another error that
-      // we need to respect.
-    } else {
-      this.setDelay(30_000);
-    }
-  }
-
-  private setDelay(delay: number) {
-    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
-  }
-}

package/src/replication/ReplicationEngine.ts

@@ -0,0 +1,43 @@
+import { AbstractReplicator } from './AbstractReplicator.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export class ReplicationEngine {
+  private readonly replicators: Map<string, AbstractReplicator> = new Map();
+
+  /**
+   *  Register a Replicator with the engine
+   *
+   *  @param replicator
+   */
+  public register(replicator: AbstractReplicator) {
+    if (this.replicators.has(replicator.id)) {
+      throw new Error(`Replicator with id: ${replicator.id} already registered`);
+    }
+    logger.info(`Successfully registered Replicator ${replicator.id} with ReplicationEngine`);
+    this.replicators.set(replicator.id, replicator);
+  }
+
+  /**
+   *  Start replication on all managed Replicators
+   */
+  public start(): void {
+    logger.info('Starting Replication Engine...');
+    for (const replicator of this.replicators.values()) {
+      logger.info(`Starting Replicator: ${replicator.id}`);
+      replicator.start();
+    }
+    logger.info('Successfully started Replication Engine.');
+  }
+
+  /**
+   *  Stop replication on all managed Replicators
+   */
+  public async shutDown(): Promise<void> {
+    logger.info('Shutting down Replication Engine...');
+    for (const replicator of this.replicators.values()) {
+      logger.info(`Stopping Replicator: ${replicator.id}`);
+      await replicator.stop();
+    }
+    logger.info('Successfully shut down Replication Engine.');
+  }
+}

package/src/replication/ReplicationModule.ts

@@ -0,0 +1,101 @@
+import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
+import * as t from 'ts-codec';
+
+import * as types from '@powersync/service-types';
+import * as api from '../api/api-index.js';
+import * as modules from '../modules/modules-index.js';
+import * as system from '../system/system-index.js';
+import { schema } from '@powersync/lib-services-framework';
+import { AbstractReplicator } from './AbstractReplicator.js';
+
+export interface ReplicationModuleOptions extends modules.AbstractModuleOptions {
+  type: string;
+  configSchema: t.AnyCodec;
+}
+
+/**
+ *  A replication module describes all the functionality that PowerSync requires to
+ *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
+ */
+export abstract class ReplicationModule<TConfig extends DataSourceConfig> extends modules.AbstractModule {
+  protected type: string;
+  protected configSchema: t.AnyCodec;
+  protected decodedConfig: TConfig | undefined;
+
+  /**
+   * @protected
+   * @param options
+   */
+  protected constructor(options: ReplicationModuleOptions) {
+    super(options);
+    this.type = options.type;
+    this.configSchema = options.configSchema;
+  }
+
+  /**
+   *  Create the RouteAPI adapter for the DataSource required to service the sync API
+   *  endpoints.
+   */
+  protected abstract createRouteAPIAdapter(): api.RouteAPI;
+
+  /**
+   *  Create the Replicator to be used by the ReplicationEngine.
+   */
+  protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
+
+  /**
+   *  Register this module's Replicators and RouteAPI adapters if the required configuration is present.
+   */
+  public async initialize(context: system.ServiceContext): Promise<void> {
+    if (!context.configuration.connections) {
+      // No data source configuration found in the config skip for now
+      return;
+    }
+
+    const matchingConfig = context.configuration.connections.filter((dataSource) => dataSource.type === this.type);
+    if (!matchingConfig.length) {
+      // This module is needed given the config
+      return;
+    }
+
+    if (matchingConfig.length > 1) {
+      this.logger.warning(
+        `Multiple data sources of type ${this.type} found in the configuration. Only the first will be used.`
+      );
+    }
+
+    try {
+      const baseMatchingConfig = matchingConfig[0] as TConfig;
+      // If validation fails, log the error and continue, no replication will happen for this data source
+      this.validateConfig(baseMatchingConfig);
+      this.decodedConfig = this.configSchema.decode(baseMatchingConfig);
+
+      context.replicationEngine?.register(this.createReplicator(context));
+      context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
+    } catch (e) {
+      this.logger.error('Failed to initialize.', e);
+    }
+  }
+
+  private validateConfig(config: TConfig): void {
+    const validator = schema
+      .parseJSONSchema(
+        // This generates a schema for the encoded form of the codec
+        t.generateJSONSchema(this.configSchema, {
+          allowAdditional: true,
+          parsers: [types.configFile.portParser]
+        })
+      )
+      .validator();
+
+    const valid = validator.validate(config);
+
+    if (!valid.valid) {
+      throw new Error(`Failed to validate Module ${this.name} configuration: ${valid.errors.join(', ')}`);
+    }
+  }
+
+  protected getDefaultId(dataSourceName: string): string {
+    return `${this.type}-${dataSourceName}`;
+  }
+}

package/src/replication/replication-index.ts

@@ -1,7 +1,5 @@
 export * from './ErrorRateLimiter.js';
-export * from './PgRelation.js';
-export * from './util.js';
-export * from './WalConnection.js';
-export * from './WalStream.js';
-export * from './WalStreamManager.js';
-export * from './WalStreamRunner.js';
+export * from './AbstractReplicationJob.js';
+export * from './AbstractReplicator.js';
+export * from './ReplicationEngine.js';
+export * from './ReplicationModule.js';

package/src/routes/RouterEngine.ts

@@ -0,0 +1,120 @@
+import { logger } from '@powersync/lib-services-framework';
+
+import * as api from '../api/api-index.js';
+
+import { ADMIN_ROUTES } from './endpoints/admin.js';
+import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { syncStreamReactive } from './endpoints/socket-route.js';
+import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
+import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
+import { SocketRouteGenerator } from './router-socket.js';
+import { RouteDefinition } from './router.js';
+
+export type RouterSetupResponse = {
+  onShutdown: () => Promise<void>;
+};
+
+export type RouterEngineRoutes = {
+  api_routes: RouteDefinition[];
+  stream_routes: RouteDefinition[];
+  socket_routes: SocketRouteGenerator[];
+};
+
+export type RouterSetup = (routes: RouterEngineRoutes) => Promise<RouterSetupResponse>;
+
+/**
+ *  Serves as a registry from which SyncAPIs can be retrieved based on Replication DataSource type
+ *  Initially only one SyncAPI per DataSource type is supported
+ */
+export class RouterEngine {
+  closed: boolean;
+  routes: RouterEngineRoutes;
+
+  protected stopHandlers: Set<() => void>;
+
+  /**
+   * A final cleanup handler to be executed after all stopHandlers
+   */
+  protected cleanupHandler: (() => Promise<void>) | null;
+
+  private api: api.RouteAPI | null;
+
+  constructor() {
+    this.api = null;
+    this.stopHandlers = new Set();
+    this.cleanupHandler = null;
+    this.closed = false;
+
+    // Default routes
+    this.routes = {
+      api_routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES],
+      stream_routes: [...SYNC_STREAM_ROUTES],
+      socket_routes: [syncStreamReactive]
+    };
+  }
+
+  public registerAPI(api: api.RouteAPI) {
+    if (this.api) {
+      logger.warn('A RouteAPI has already been registered. Overriding existing implementation');
+    }
+
+    this.api = api;
+  }
+
+  public getAPI(): api.RouteAPI {
+    if (!this.api) {
+      throw new Error('No RouteAPI adapter has been registered yet.');
+    }
+    return this.api;
+  }
+
+  /**
+   * Starts the router given the configuration provided
+   */
+  async start(setup: RouterSetup) {
+    logger.info('Starting Router Engine...');
+    const { onShutdown } = await setup(this.routes);
+    this.cleanupHandler = onShutdown;
+    logger.info('Successfully started Router Engine.');
+  }
+
+  /**
+   * Runs all stop handlers then final cleanup.
+   */
+  async shutDown() {
+    logger.info('Shutting down Router Engine...');
+    // Close open streams, so that they don't block the server from closing.
+    // Note: This does not work well when streaming requests are queued. In that case, the server still doesn't
+    // close in the 30-second timeout.
+    this.closed = true;
+
+    logger.info(`Closing ${this.stopHandlers.size} streams.`);
+    for (let handler of this.stopHandlers) {
+      handler();
+    }
+
+    logger.info(`Running cleanup.`);
+
+    // Typically closes the server
+    await this.cleanupHandler?.();
+
+    // Close the api handlers
+    await this.api?.shutdown();
+    logger.info('Successfully shut down Router Engine.');
+  }
+
+  /**
+   * Add a stop handler callback to be executed when the router engine is being
+   * shutdown.
+   */
+  addStopHandler(handler: () => void): () => void {
+    if (this.closed) {
+      handler();
+      return () => {};
+    }
+    this.stopHandlers.add(handler);
+    return () => {
+      this.stopHandlers.delete(handler);
+    };
+  }
+}