@powersync/service-core 0.0.2

package/src/replication/WalStreamManager.ts

@@ -0,0 +1,214 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import * as micro from '@journeyapps-platform/micro';
+import { hrtime } from 'node:process';
+
+import * as storage from '@/storage/storage-index.js';
+import * as util from '@/util/util-index.js';
+
+import { DefaultErrorRateLimiter } from './ErrorRateLimiter.js';
+import { touch } from './WalStream.js';
+import { WalStreamRunner } from './WalStreamRunner.js';
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+
+// 5 minutes
+const PING_INTERVAL = 1_000_000_000n * 300n;
+
+export class WalStreamManager {
+  private streams = new Map<number, WalStreamRunner>();
+
+  private system: CorePowerSyncSystem;
+
+  private stopped = false;
+
+  // First ping is only after 5 minutes, not when starting
+  private lastPing = hrtime.bigint();
+
+  private storage: storage.BucketStorageFactory;
+
+  /**
+   * This limits the effect of retries when there is a persistent issue.
+   */
+  private rateLimiter = new DefaultErrorRateLimiter();
+
+  constructor(system: CorePowerSyncSystem) {
+    this.system = system;
+    this.storage = system.storage;
+  }
+
+  start() {
+    this.runLoop().catch((e) => {
+      micro.logger.error(`Fatal WalStream error`, e);
+      util.captureException(e);
+      setTimeout(() => {
+        process.exit(1);
+      }, 1000);
+    });
+  }
+
+  async stop() {
+    this.stopped = true;
+    let promises: Promise<void>[] = [];
+    for (let stream of this.streams.values()) {
+      promises.push(stream.stop());
+    }
+    await Promise.all(promises);
+  }
+
+  private async runLoop() {
+    const configured_sync_rules = await util.loadSyncRules(this.system.config);
+    let configured_lock: storage.ReplicationLock | undefined = undefined;
+    if (configured_sync_rules != null) {
+      micro.logger.info('Loading sync rules from configuration');
+      try {
+        // Configure new sync rules, if it has changed.
+        // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+        const { updated, persisted_sync_rules, lock } = await this.storage.configureSyncRules(configured_sync_rules!, {
+          lock: true
+        });
+        if (lock) {
+          configured_lock = lock;
+        }
+      } catch (e) {
+        // Log, but continue with previous sync rules
+        micro.logger.error(`Failed to load sync rules from configuration`, e);
+      }
+    } else {
+      micro.logger.info('No sync rules configured - configure via API');
+    }
+    while (!this.stopped) {
+      await touch();
+      try {
+        const pool = this.system.pgwire_pool;
+        if (pool) {
+          await this.refresh({ configured_lock });
+          // The lock is only valid on the first refresh.
+          configured_lock = undefined;
+
+          // TODO: Ping on all connections when we have multiple
+          // Perhaps WalStreamRunner would be a better place to do pings?
+          // We don't ping while in error retry back-off, to avoid having too
+          // many authentication failures.
+          if (this.rateLimiter.mayPing()) {
+            await this.ping(pool);
+          }
+        }
+      } catch (e) {
+        micro.logger.error(`Failed to refresh wal streams`, e);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+    }
+  }
+
+  /**
+   * Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
+   *
+   * The old WAL files are only deleted once no replication slot still references it.
+   *
+   * Unfortunately, when there are no changes to the db, the database creates new WAL files without the replication slot
+   * advancing**.
+   *
+   * As a workaround, we write a new message every couple of minutes, to make sure that the replication slot advances.
+   *
+   * **This may be a bug in pgwire or how we're using it.
+   */
+  private async ping(db: pgwire.PgClient) {
+    const now = hrtime.bigint();
+    if (now - this.lastPing >= PING_INTERVAL) {
+      try {
+        await db.query(`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`);
+      } catch (e) {
+        micro.logger.warn(`Failed to ping`, e);
+      }
+      this.lastPing = now;
+    }
+  }
+
+  private async refresh(options?: { configured_lock?: storage.ReplicationLock }) {
+    if (this.stopped) {
+      return;
+    }
+
+    let configured_lock = options?.configured_lock;
+
+    const existingStreams = new Map<number, WalStreamRunner>(this.streams.entries());
+    const replicating = await this.storage.getReplicatingSyncRules();
+    const newStreams = new Map<number, WalStreamRunner>();
+    for (let syncRules of replicating) {
+      const existing = existingStreams.get(syncRules.id);
+      if (existing && !existing.stopped) {
+        // No change
+        existingStreams.delete(syncRules.id);
+        newStreams.set(syncRules.id, existing);
+      } else if (existing && existing.stopped) {
+        // Stopped (e.g. fatal error, slot rename).
+        // Remove from the list. Next refresh call will restart the stream.
+        existingStreams.delete(syncRules.id);
+      } else {
+        // New (or resume after restart)
+        try {
+          let lock: storage.ReplicationLock;
+          if (configured_lock?.sync_rules_id == syncRules.id) {
+            lock = configured_lock;
+          } else {
+            lock = await syncRules.lock();
+          }
+          const parsed = syncRules.parsed();
+          const storage = this.storage.getInstance(parsed);
+          const stream = new WalStreamRunner({
+            factory: this.storage,
+            storage: storage,
+            source_db: this.system.config.connection!,
+            lock,
+            rateLimiter: this.rateLimiter
+          });
+          newStreams.set(syncRules.id, stream);
+          stream.start();
+        } catch (e) {
+          // Could be a sync rules parse error,
+          // for example from stricter validation that was added.
+          // This will be retried every couple of seconds.
+          // When new (valid) sync rules are deployed and processed, this one be disabled.
+          micro.logger.error(`Failed to start replication for ${syncRules.slot_name}`, e);
+        }
+      }
+    }
+
+    this.streams = newStreams;
+
+    // TODO: Should this termination be happening in the "background" instead?
+    // That becomes tricky to manage
+
+    for (let stream of existingStreams.values()) {
+      // Old - stop and remove.
+      try {
+        await stream.terminate();
+      } catch (e) {
+        // This will be retried
+        micro.logger.warn(`Failed to terminate ${stream.slot_name}`, e);
+      }
+    }
+
+    // Sync rules stopped previously or by a different process.
+    const stopped = await this.storage.getStoppedSyncRules();
+    for (let syncRules of stopped) {
+      try {
+        const lock = await syncRules.lock();
+        try {
+          const parsed = syncRules.parsed();
+          const storage = this.storage.getInstance(parsed);
+          const stream = new WalStreamRunner({
+            factory: this.storage,
+            storage: storage,
+            source_db: this.system.config.connection!,
+            lock
+          });
+          await stream.terminate();
+        } finally {
+          await lock.release();
+        }
+      } catch (e) {
+        micro.logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
+      }
+    }
+  }
+}

package/src/replication/WalStreamRunner.ts

@@ -0,0 +1,180 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import * as micro from '@journeyapps-platform/micro';
+
+import * as storage from '@/storage/storage-index.js';
+import * as util from '@/util/util-index.js';
+
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+import { MissingReplicationSlotError, WalStream } from './WalStream.js';
+import { ResolvedConnection } from '../util/config/types.js';
+
+export interface WalStreamRunnerOptions {
+  factory: storage.BucketStorageFactory;
+  storage: storage.SyncRulesBucketStorage;
+  source_db: ResolvedConnection;
+  lock: storage.ReplicationLock;
+  rateLimiter?: ErrorRateLimiter;
+}
+
+export class WalStreamRunner {
+  private abortController = new AbortController();
+
+  private runPromise?: Promise<void>;
+
+  private connections: util.PgManager | null = null;
+
+  private rateLimiter?: ErrorRateLimiter;
+
+  constructor(public options: WalStreamRunnerOptions) {
+    this.rateLimiter = options.rateLimiter;
+  }
+
+  start() {
+    this.runPromise = this.run();
+  }
+
+  get slot_name() {
+    return this.options.storage.slot_name;
+  }
+
+  get stopped() {
+    return this.abortController.signal.aborted;
+  }
+
+  async run() {
+    try {
+      await this.replicateLoop();
+    } catch (e) {
+      // Fatal exception
+      util.captureException(e, {
+        metadata: {
+          replication_slot: this.slot_name
+        }
+      });
+      micro.logger.error(`Replication failed on ${this.slot_name}`, e);
+
+      if (e instanceof MissingReplicationSlotError) {
+        // This stops replication on this slot, and creates a new slot
+        await this.options.storage.factory.slotRemoved(this.slot_name);
+      }
+    } finally {
+      this.abortController.abort();
+    }
+    await this.options.lock.release();
+  }
+
+  async replicateLoop() {
+    while (!this.stopped) {
+      await this.replicateOnce();
+
+      if (!this.stopped) {
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+      }
+    }
+  }
+
+  async replicateOnce() {
+    // New connections on every iteration (every error with retry),
+    // otherwise we risk repeating errors related to the connection,
+    // such as caused by cached PG schemas.
+    let connections = new util.PgManager(this.options.source_db, {
+      // Pool connections are only used intermittently.
+      idleTimeout: 30_000,
+      maxSize: 2
+    });
+    this.connections = connections;
+    try {
+      await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
+      if (this.stopped) {
+        return;
+      }
+      const stream = new WalStream({
+        abort_signal: this.abortController.signal,
+        factory: this.options.factory,
+        storage: this.options.storage,
+        connections
+      });
+      await stream.replicate();
+    } catch (e) {
+      micro.logger.error(`Replication error`, e);
+      if (e.cause != null) {
+        // Example:
+        // PgError.conn_ended: Unable to do postgres query on ended connection
+        //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
+        //     at stream.next (<anonymous>)
+        //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
+        //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
+        //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
+        //     ...
+        //   cause: TypeError: match is not iterable
+        //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
+        //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
+        //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
+        //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
+        //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
+        //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
+        //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
+        //   [Symbol(pg.ErrorCode)]: 'conn_ended',
+        //   [Symbol(pg.ErrorResponse)]: undefined
+        // }
+        // Without this additional log, the cause would not be visible in the logs.
+        micro.logger.error(`cause`, e.cause);
+      }
+      if (e instanceof MissingReplicationSlotError) {
+        throw e;
+      } else {
+        // Report the error if relevant, before retrying
+        util.captureException(e, {
+          metadata: {
+            replication_slot: this.slot_name
+          }
+        });
+        // This sets the retry delay
+        this.rateLimiter?.reportError(e);
+      }
+    } finally {
+      this.connections = null;
+      if (connections != null) {
+        await connections.end();
+      }
+    }
+  }
+
+  /**
+   * This will also release the lock if start() was called earlier.
+   */
+  async stop(options?: { force?: boolean }) {
+    micro.logger.info(`${this.slot_name} Stopping replication`);
+    // End gracefully
+    this.abortController.abort();
+
+    if (options?.force) {
+      // destroy() is more forceful.
+      await this.connections?.destroy();
+    }
+    await this.runPromise;
+  }
+
+  /**
+   * Terminate this replication stream. This drops the replication slot and deletes the replication data.
+   *
+   * Stops replication if needed.
+   */
+  async terminate(options?: { force?: boolean }) {
+    micro.logger.info(`${this.slot_name} Terminating replication`);
+    await this.stop(options);
+
+    const slotName = this.slot_name;
+    const db = await pgwire.connectPgWire(this.options.source_db, { type: 'standard' });
+    try {
+      await db.query({
+        statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
+        params: [{ type: 'varchar', value: slotName }]
+      });
+    } finally {
+      await db.end();
+    }
+
+    await this.options.storage.terminate();
+  }
+}

package/src/replication/replication-index.ts

@@ -0,0 +1,7 @@
+export * from './ErrorRateLimiter.js';
+export * from './PgRelation.js';
+export * from './util.js';
+export * from './WalConnection.js';
+export * from './WalStream.js';
+export * from './WalStreamManager.js';
+export * from './WalStreamRunner.js';

package/src/replication/util.ts

@@ -0,0 +1,76 @@
+import * as pgwire from '@powersync/service-jpgwire';
+
+import * as util from '@/util/util-index.js';
+import { ReplicationColumn, ReplicationIdentity } from './PgRelation.js';
+
+export interface ReplicaIdentityResult {
+  columns: ReplicationColumn[];
+  replicationIdentity: ReplicationIdentity;
+}
+
+export async function getPrimaryKeyColumns(
+  db: pgwire.PgClient,
+  relationId: number,
+  mode: 'primary' | 'replident'
+): Promise<ReplicationColumn[]> {
+  const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`;
+  const attrRows = await util.retriedQuery(db, {
+    statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
+                                    FROM pg_index i
+                                             JOIN pg_attribute a
+                                                  ON a.attrelid = i.indrelid
+                                                      AND a.attnum = ANY (i.indkey)
+                                    WHERE i.indrelid = $1::oid
+                                      AND ${indexFlag}
+                                      AND a.attnum > 0
+                                      ORDER BY a.attnum`,
+    params: [{ value: relationId, type: 'int4' }]
+  });
+
+  return attrRows.rows.map((row) => {
+    return { name: row[0] as string, typeOid: row[1] as number };
+  });
+}
+
+export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise<ReplicationColumn[]> {
+  const attrRows = await util.retriedQuery(db, {
+    statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
+                                    FROM pg_attribute a
+                                    WHERE a.attrelid = $1::oid
+                                      AND attnum > 0
+                                    ORDER BY a.attnum`,
+    params: [{ type: 'varchar', value: relationId }]
+  });
+  return attrRows.rows.map((row) => {
+    return { name: row[0] as string, typeOid: row[1] as number };
+  });
+}
+
+export async function getReplicationIdentityColumns(
+  db: pgwire.PgClient,
+  relationId: number
+): Promise<ReplicaIdentityResult> {
+  const rows = await util.retriedQuery(db, {
+    statement: `SELECT CASE relreplident
+        WHEN 'd' THEN 'default'
+        WHEN 'n' THEN 'nothing'
+        WHEN 'f' THEN 'full'
+        WHEN 'i' THEN 'index'
+     END AS replica_identity
+FROM pg_class
+WHERE oid = $1::oid LIMIT 1`,
+    params: [{ type: 'int8', value: relationId }]
+  });
+  const idType: string = rows.rows[0]?.[0];
+  if (idType == 'nothing' || idType == null) {
+    return { replicationIdentity: 'nothing', columns: [] };
+  } else if (idType == 'full') {
+    return { replicationIdentity: 'full', columns: await getAllColumns(db, relationId) };
+  } else if (idType == 'default') {
+    return { replicationIdentity: 'default', columns: await getPrimaryKeyColumns(db, relationId, 'primary') };
+  } else if (idType == 'index') {
+    return { replicationIdentity: 'index', columns: await getPrimaryKeyColumns(db, relationId, 'replident') };
+  } else {
+    return { replicationIdentity: 'nothing', columns: [] };
+  }
+}