@powersync/service-core 0.8.7 → 0.9.0

package/src/metrics/Metrics.ts

@@ -1,13 +1,11 @@
 import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
-import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '../util/util-index.js';
-import * as storage from '../storage/storage-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { Resource } from '@opentelemetry/resources';
+import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { logger } from '@powersync/lib-services-framework';
+import * as storage from '../storage/storage-index.js';
+import * as util from '../util/util-index.js';
 
 export interface MetricsOptions {
   disable_telemetry_sharing: boolean;
@@ -202,7 +200,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
     this.concurrent_connections.add(0);
   }
 
-  public configureReplicationMetrics(system: CorePowerSyncSystem) {
+  public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) {
     // Rate limit collection of these stats, since it may be an expensive query
     const MINIMUM_INTERVAL = 60_000;
 
@@ -211,7 +209,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
 
     function getMetrics() {
       if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
-        cachedRequest = system.storage.getStorageMetrics().catch((e) => {
+        cachedRequest = bucketStorage.getStorageMetrics().catch((e) => {
           logger.error(`Failed to get storage metrics`, e);
           return null;
         });
@@ -240,14 +238,6 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
         result.observe(metrics.replication_size_bytes);
       }
     });
-
-    const class_scoped_data_replicated_bytes = this.data_replicated_bytes;
-    // Record replicated bytes using global jpgwire metrics.
-    jpgwire.setMetricsRecorder({
-      addBytesRead(bytes) {
-        class_scoped_data_replicated_bytes.add(bytes);
-      }
-    });
   }
 
   public async getMetricValueForTests(name: string): Promise<number | undefined> {

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -2,8 +2,11 @@ import * as mongo from '../../../db/mongo.js';
 import * as storage from '../../../storage/storage-index.js';
 import * as utils from '../../../util/util-index.js';
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+
+  const config = await utils.loadConfig(runner_config);
+
   const database = storage.createPowerSyncMongo(config.storage);
   await mongo.waitForAuth(database.db);
   try {
@@ -20,8 +23,10 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+
   const database = storage.createPowerSyncMongo(config.storage);
   try {
     if (await database.bucket_parameters.indexExists('lookup')) {

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -23,9 +23,11 @@ interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   auto_activate?: boolean;
 }
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
   const db = storage.createPowerSyncMongo(config.storage);
+
   await mongo.waitForAuth(db.db);
   try {
     // We keep the old flags for existing deployments still shutting down.
@@ -68,8 +70,9 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
 
   const db = storage.createPowerSyncMongo(config.storage);
   try {

package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts

@@ -1,8 +1,9 @@
 import * as storage from '../../../storage/storage-index.js';
 import * as utils from '../../../util/util-index.js';
 
-export const up = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
   const db = storage.createPowerSyncMongo(config.storage);
 
   try {
@@ -17,8 +18,9 @@ export const up = async (context?: utils.MigrationContext) => {
   }
 };
 
-export const down = async (context?: utils.MigrationContext) => {
-  const config = await utils.loadConfig(context?.runner_config);
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
 
   const db = storage.createPowerSyncMongo(config.storage);
 

package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts

@@ -0,0 +1,37 @@
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
+
+const INDEX_NAME = 'user_sync_rule_unique';
+
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    await db.custom_write_checkpoints.createIndex(
+      {
+        user_id: 1,
+        sync_rules_id: 1
+      },
+      { name: INDEX_NAME, unique: true }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
+      await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/migrations/migrations.ts

@@ -2,13 +2,13 @@ import * as fs from 'fs/promises';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 
+import { logger } from '@powersync/lib-services-framework';
 import * as db from '../db/db-index.js';
-import * as util from '../util/util-index.js';
 import * as locks from '../locks/locks-index.js';
+import * as util from '../util/util-index.js';
 import { Direction } from './definitions.js';
-import { createMongoMigrationStore } from './store/migration-store.js';
 import { execute, writeLogsToStore } from './executor.js';
-import { logger } from '@powersync/lib-services-framework';
+import { createMongoMigrationStore } from './store/migration-store.js';
 
 const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
 const MONGO_LOCK_PROCESS = 'migrations';
@@ -23,18 +23,23 @@ export type MigrationOptions = {
   runner_config: util.RunnerConfig;
 };
 
+export type AutomaticMigrationParams = {
+  config: util.ResolvedPowerSyncConfig;
+  runner_config: util.RunnerConfig;
+};
+
 /**
  * Loads migrations and injects a custom context for loading the specified
  * runner configuration.
  */
-const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) => {
+const loadMigrations = async (dir: string, runnerConfig: util.RunnerConfig) => {
   const files = await fs.readdir(dir);
   const migrations = files.filter((file) => {
     return path.extname(file) === '.js';
   });
 
   const context: util.MigrationContext = {
-    runner_config
+    runner_config: runnerConfig
   };
 
   return await Promise.all(
@@ -55,14 +60,13 @@ const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) =>
 export const migrate = async (options: MigrationOptions) => {
   const { direction, runner_config } = options;
 
+  const config = await util.loadConfig(runner_config);
+  const { storage } = config;
   /**
    * Try and get Mongo from config file.
    * But this might not be available in Journey Micro as we use the standard Mongo.
    */
 
-  const config = await util.loadConfig(runner_config);
-  const { storage } = config;
-
   const client = db.mongo.createMongoClient(storage);
   logger.info('Connecting to MongoDB');
   await client.connect();
@@ -124,3 +128,15 @@ export const migrate = async (options: MigrationOptions) => {
     logger.info('Done with migrations');
   }
 };
+
+/**
+ * Ensures automatic migrations are executed
+ */
+export const ensureAutomaticMigrations = async (params: AutomaticMigrationParams) => {
+  if (!params.config.migrations?.disable_auto_migration) {
+    await migrate({
+      direction: Direction.Up,
+      runner_config: params.runner_config
+    });
+  }
+};

package/src/modules/AbstractModule.ts

@@ -0,0 +1,37 @@
+import { ServiceContextContainer } from '../system/ServiceContext.js';
+import { logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+import { PersistedSyncRulesContent } from '../storage/BucketStorage.js';
+
+export interface TearDownOptions {
+  /**
+   *  If required, tear down any configuration/state for the specific sync rules
+   */
+  syncRules?: PersistedSyncRulesContent[];
+}
+
+export interface AbstractModuleOptions {
+  name: string;
+}
+
+export abstract class AbstractModule {
+  protected logger: winston.Logger;
+
+  protected constructor(protected options: AbstractModuleOptions) {
+    this.logger = logger.child({ name: `Module:${options.name}` });
+  }
+
+  /**
+   *  Initialize the module using any required services from the ServiceContext
+   */
+  public abstract initialize(context: ServiceContextContainer): Promise<void>;
+
+  /**
+   *  Permanently clean up and dispose of any configuration or state for this module.
+   */
+  public abstract teardown(options: TearDownOptions): Promise<void>;
+
+  public get name() {
+    return this.options.name;
+  }
+}

package/src/modules/ModuleManager.ts

@@ -0,0 +1,34 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as system from '../system/system-index.js';
+import { AbstractModule, TearDownOptions } from './AbstractModule.js';
+/**
+ *  The module manager keeps track of activated modules
+ */
+export class ModuleManager {
+  private readonly modules: Map<string, AbstractModule> = new Map();
+
+  public register(modules: AbstractModule[]) {
+    for (const module of modules) {
+      if (this.modules.has(module.name)) {
+        logger.warn(`Module ${module.name} already registered, skipping.`);
+        continue;
+      }
+      this.modules.set(module.name, module);
+      logger.info(`Successfully registered Module ${module.name}.`);
+    }
+  }
+
+  async initialize(serviceContext: system.ServiceContextContainer) {
+    logger.info(`Initializing modules...`);
+    for (const module of this.modules.values()) {
+      await module.initialize(serviceContext);
+    }
+    logger.info(`Successfully Initialized modules.`);
+  }
+
+  async tearDown(options: TearDownOptions) {
+    for (const module of this.modules.values()) {
+      await module.teardown(options);
+    }
+  }
+}

package/src/modules/modules-index.ts

@@ -0,0 +1,2 @@
+export * from './ModuleManager.js';
+export * from './AbstractModule.js';

package/src/replication/AbstractReplicationJob.ts

@@ -0,0 +1,79 @@
+import { container, logger } from '@powersync/lib-services-framework';
+import winston from 'winston';
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+
+export interface AbstractReplicationJobOptions {
+  id: string;
+  storage: storage.SyncRulesBucketStorage;
+  lock: storage.ReplicationLock;
+  rateLimiter: ErrorRateLimiter;
+}
+
+export abstract class AbstractReplicationJob {
+  protected logger: winston.Logger;
+  protected abortController = new AbortController();
+  protected isReplicatingPromise: Promise<void> | null = null;
+
+  protected constructor(protected options: AbstractReplicationJobOptions) {
+    this.logger = logger.child({ name: `ReplicationJob: ${this.id}` });
+  }
+
+  /**
+   *  Copy the initial data set from the data source if required and then keep it in sync.
+   */
+  abstract replicate(): Promise<void>;
+
+  /**
+   *  Ensure the connection to the data source remains active
+   */
+  abstract keepAlive(): Promise<void>;
+
+  /**
+   *  Start the replication process
+   */
+  public async start(): Promise<void> {
+    this.isReplicatingPromise = this.replicate()
+      .catch((ex) => {
+        container.reporter.captureException(ex, {
+          metadata: {
+            replicator: this.id
+          }
+        });
+        this.logger.error(`Replication failed.`, ex);
+      })
+      .finally(async () => {
+        this.abortController.abort();
+        await this.options.lock.release();
+      });
+  }
+
+  /**
+   *  Safely stop the replication process
+   */
+  public async stop(): Promise<void> {
+    this.logger.info(`Stopping replication job for sync rule iteration: ${this.storage.group_id}`);
+    this.abortController.abort();
+    await this.isReplicatingPromise;
+  }
+
+  public get id() {
+    return this.options.id;
+  }
+
+  public get storage() {
+    return this.options.storage;
+  }
+
+  protected get lock() {
+    return this.options.lock;
+  }
+
+  protected get rateLimiter() {
+    return this.options.rateLimiter;
+  }
+
+  public get isStopped(): boolean {
+    return this.abortController.signal.aborted;
+  }
+}

package/src/replication/AbstractReplicator.ts

@@ -0,0 +1,228 @@
+import { container, logger } from '@powersync/lib-services-framework';
+import { hrtime } from 'node:process';
+import winston from 'winston';
+import * as storage from '../storage/storage-index.js';
+import { StorageEngine } from '../storage/storage-index.js';
+import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
+import { AbstractReplicationJob } from './AbstractReplicationJob.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+
+// 5 minutes
+const PING_INTERVAL = 1_000_000_000n * 300n;
+
+export interface CreateJobOptions {
+  lock: storage.ReplicationLock;
+  storage: storage.SyncRulesBucketStorage;
+}
+
+export interface AbstractReplicatorOptions {
+  id: string;
+  storageEngine: StorageEngine;
+  syncRuleProvider: SyncRulesProvider;
+  /**
+   * This limits the effect of retries when there is a persistent issue.
+   */
+  rateLimiter: ErrorRateLimiter;
+}
+
+/**
+ *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
+ *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
+ *   It also handles any changes to the sync rules.
+ */
+export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
+  protected logger: winston.Logger;
+  /**
+   *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
+   *  transitioning to a new set of sync rules.
+   *  @private
+   */
+  private replicationJobs = new Map<number, T>();
+  private stopped = false;
+
+  // First ping is only after 5 minutes, not when starting
+  private lastPing = hrtime.bigint();
+
+  protected constructor(private options: AbstractReplicatorOptions) {
+    this.logger = logger.child({ name: `Replicator:${options.id}` });
+  }
+
+  abstract createJob(options: CreateJobOptions): T;
+
+  /**
+   *  Clean up any configuration or state for the specified sync rule on the datasource.
+   *  Should be a no-op if the configuration has already been cleared
+   */
+  abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
+
+  public get id() {
+    return this.options.id;
+  }
+
+  protected get storage() {
+    return this.options.storageEngine.activeBucketStorage;
+  }
+
+  protected get syncRuleProvider() {
+    return this.options.syncRuleProvider;
+  }
+
+  protected get rateLimiter() {
+    return this.options.rateLimiter;
+  }
+
+  public async start(): Promise<void> {
+    this.runLoop().catch((e) => {
+      this.logger.error('Data source fatal replication error', e);
+      container.reporter.captureException(e);
+      setTimeout(() => {
+        process.exit(1);
+      }, 1000);
+    });
+  }
+
+  public async stop(): Promise<void> {
+    this.stopped = true;
+    let promises: Promise<void>[] = [];
+    for (const job of this.replicationJobs.values()) {
+      promises.push(job.stop());
+    }
+    await Promise.all(promises);
+  }
+
+  private async runLoop() {
+    const syncRules = await this.syncRuleProvider.get();
+    let configuredLock: storage.ReplicationLock | undefined = undefined;
+    if (syncRules != null) {
+      this.logger.info('Loaded sync rules');
+      try {
+        // Configure new sync rules, if they have changed.
+        // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+        const { lock } = await this.storage.configureSyncRules(syncRules, {
+          lock: true
+        });
+        if (lock) {
+          configuredLock = lock;
+        }
+      } catch (e) {
+        // Log, but continue with previous sync rules
+        this.logger.error(`Failed to update sync rules from configuration`, e);
+      }
+    } else {
+      this.logger.info('No sync rules configured - configure via API');
+    }
+    while (!this.stopped) {
+      await container.probes.touch();
+      try {
+        await this.refresh({ configured_lock: configuredLock });
+        // The lock is only valid on the first refresh.
+        configuredLock = undefined;
+
+        // Ensure that the replication jobs' connections are kept alive.
+        // We don't ping while in error retry back-off, to avoid having too failures.
+        if (this.rateLimiter.mayPing()) {
+          const now = hrtime.bigint();
+          if (now - this.lastPing >= PING_INTERVAL) {
+            for (const activeJob of this.replicationJobs.values()) {
+              await activeJob.keepAlive();
+            }
+
+            this.lastPing = now;
+          }
+        }
+      } catch (e) {
+        this.logger.error('Failed to refresh replication jobs', e);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+    }
+  }
+
+  private async refresh(options?: { configured_lock?: storage.ReplicationLock }) {
+    if (this.stopped) {
+      return;
+    }
+
+    let configuredLock = options?.configured_lock;
+
+    const existingJobs = new Map<number, T>(this.replicationJobs.entries());
+    const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
+    const newJobs = new Map<number, T>();
+    for (let syncRules of replicatingSyncRules) {
+      const existingJob = existingJobs.get(syncRules.id);
+      if (existingJob && !existingJob.isStopped) {
+        // No change
+        existingJobs.delete(syncRules.id);
+        newJobs.set(syncRules.id, existingJob);
+      } else if (existingJob && existingJob.isStopped) {
+        // Stopped (e.g. fatal error).
+        // Remove from the list. Next refresh call will restart the job.
+        existingJobs.delete(syncRules.id);
+      } else {
+        // New sync rules were found (or resume after restart)
+        try {
+          let lock: storage.ReplicationLock;
+          if (configuredLock?.sync_rules_id == syncRules.id) {
+            lock = configuredLock;
+          } else {
+            lock = await syncRules.lock();
+          }
+          const storage = this.storage.getInstance(syncRules);
+          const newJob = this.createJob({
+            lock: lock,
+            storage: storage
+          });
+
+          newJobs.set(syncRules.id, newJob);
+          newJob.start();
+        } catch (e) {
+          // Could be a sync rules parse error,
+          // for example from stricter validation that was added.
+          // This will be retried every couple of seconds.
+          // When new (valid) sync rules are deployed and processed, this one be disabled.
+          this.logger.error('Failed to start replication for new sync rules', e);
+        }
+      }
+    }
+
+    this.replicationJobs = newJobs;
+
+    // Terminate any orphaned jobs that no longer have sync rules
+    for (let job of existingJobs.values()) {
+      // Old - stop and clean up
+      try {
+        await job.stop();
+        await this.terminateSyncRules(job.storage);
+        job.storage[Symbol.dispose]();
+      } catch (e) {
+        // This will be retried
+        this.logger.warn('Failed to terminate old replication job}', e);
+      }
+    }
+
+    // Sync rules stopped previously or by a different process.
+    const stopped = await this.storage.getStoppedSyncRules();
+    for (let syncRules of stopped) {
+      try {
+        using syncRuleStorage = this.storage.getInstance(syncRules);
+        await this.terminateSyncRules(syncRuleStorage);
+      } catch (e) {
+        this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
+      }
+    }
+  }
+
+  protected createJobId(syncRuleId: number) {
+    return `${this.id}-${syncRuleId}`;
+  }
+
+  protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
+    this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
+    try {
+      await this.cleanUp(syncRuleStorage);
+      await syncRuleStorage.terminate();
+      this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
+    } catch (e) {
+      this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
+    }
+  }
+}

package/src/replication/ErrorRateLimiter.ts

@@ -1,50 +1,6 @@
-import { setTimeout } from 'timers/promises';
-
 export interface ErrorRateLimiter {
   waitUntilAllowed(options?: { signal?: AbortSignal }): Promise<void>;
   reportError(e: any): void;
 
   mayPing(): boolean;
 }
-
-export class DefaultErrorRateLimiter implements ErrorRateLimiter {
-  nextAllowed: number = Date.now();
-
-  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
-    const delay = Math.max(0, this.nextAllowed - Date.now());
-    // Minimum delay between connections, even without errors
-    this.setDelay(500);
-    await setTimeout(delay, undefined, { signal: options?.signal });
-  }
-
-  mayPing(): boolean {
-    return Date.now() >= this.nextAllowed;
-  }
-
-  reportError(e: any): void {
-    const message = (e.message as string) ?? '';
-    if (message.includes('password authentication failed')) {
-      // Wait 15 minutes, to avoid triggering Supabase's fail2ban
-      this.setDelay(900_000);
-    } else if (message.includes('ENOTFOUND')) {
-      // DNS lookup issue - incorrect URI or deleted instance
-      this.setDelay(120_000);
-    } else if (message.includes('ECONNREFUSED')) {
-      // Could be fail2ban or similar
-      this.setDelay(120_000);
-    } else if (
-      message.includes('Unable to do postgres query on ended pool') ||
-      message.includes('Postgres unexpectedly closed connection')
-    ) {
-      // Connection timed out - ignore / immediately retry
-      // We don't explicitly set the delay to 0, since there could have been another error that
-      // we need to respect.
-    } else {
-      this.setDelay(30_000);
-    }
-  }
-
-  private setDelay(delay: number) {
-    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
-  }
-}