@powersync/service-core 0.8.8 → 0.10.0

package/src/replication/WalStreamManager.ts

@@ -1,213 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { hrtime } from 'node:process';
-
-import * as storage from '../storage/storage-index.js';
-import * as util from '../util/util-index.js';
-
-import { DefaultErrorRateLimiter } from './ErrorRateLimiter.js';
-import { WalStreamRunner } from './WalStreamRunner.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-import { container, logger } from '@powersync/lib-services-framework';
-
-// 5 minutes
-const PING_INTERVAL = 1_000_000_000n * 300n;
-
-export class WalStreamManager {
-  private streams = new Map<number, WalStreamRunner>();
-
-  private system: CorePowerSyncSystem;
-
-  private stopped = false;
-
-  // First ping is only after 5 minutes, not when starting
-  private lastPing = hrtime.bigint();
-
-  private storage: storage.BucketStorageFactory;
-
-  /**
-   * This limits the effect of retries when there is a persistent issue.
-   */
-  private rateLimiter = new DefaultErrorRateLimiter();
-
-  constructor(system: CorePowerSyncSystem) {
-    this.system = system;
-    this.storage = system.storage;
-  }
-
-  start() {
-    this.runLoop().catch((e) => {
-      logger.error(`Fatal WalStream error`, e);
-      container.reporter.captureException(e);
-      setTimeout(() => {
-        process.exit(1);
-      }, 1000);
-    });
-  }
-
-  async stop() {
-    this.stopped = true;
-    let promises: Promise<void>[] = [];
-    for (let stream of this.streams.values()) {
-      promises.push(stream.stop());
-    }
-    await Promise.all(promises);
-  }
-
-  private async runLoop() {
-    const configured_sync_rules = await util.loadSyncRules(this.system.config);
-    let configured_lock: storage.ReplicationLock | undefined = undefined;
-    if (configured_sync_rules != null) {
-      logger.info('Loading sync rules from configuration');
-      try {
-        // Configure new sync rules, if it has changed.
-        // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
-        const { updated, persisted_sync_rules, lock } = await this.storage.configureSyncRules(configured_sync_rules!, {
-          lock: true
-        });
-        if (lock) {
-          configured_lock = lock;
-        }
-      } catch (e) {
-        // Log, but continue with previous sync rules
-        logger.error(`Failed to load sync rules from configuration`, e);
-      }
-    } else {
-      logger.info('No sync rules configured - configure via API');
-    }
-    while (!this.stopped) {
-      await container.probes.touch();
-      try {
-        const pool = this.system.pgwire_pool;
-        if (pool) {
-          await this.refresh({ configured_lock });
-          // The lock is only valid on the first refresh.
-          configured_lock = undefined;
-
-          // TODO: Ping on all connections when we have multiple
-          // Perhaps WalStreamRunner would be a better place to do pings?
-          // We don't ping while in error retry back-off, to avoid having too
-          // many authentication failures.
-          if (this.rateLimiter.mayPing()) {
-            await this.ping(pool);
-          }
-        }
-      } catch (e) {
-        logger.error(`Failed to refresh wal streams`, e);
-      }
-      await new Promise((resolve) => setTimeout(resolve, 5000));
-    }
-  }
-
-  /**
-   * Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
-   *
-   * The old WAL files are only deleted once no replication slot still references it.
-   *
-   * Unfortunately, when there are no changes to the db, the database creates new WAL files without the replication slot
-   * advancing**.
-   *
-   * As a workaround, we write a new message every couple of minutes, to make sure that the replication slot advances.
-   *
-   * **This may be a bug in pgwire or how we're using it.
-   */
-  private async ping(db: pgwire.PgClient) {
-    const now = hrtime.bigint();
-    if (now - this.lastPing >= PING_INTERVAL) {
-      try {
-        await db.query(`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`);
-      } catch (e) {
-        logger.warn(`Failed to ping`, e);
-      }
-      this.lastPing = now;
-    }
-  }
-
-  private async refresh(options?: { configured_lock?: storage.ReplicationLock }) {
-    if (this.stopped) {
-      return;
-    }
-
-    let configured_lock = options?.configured_lock;
-
-    const existingStreams = new Map<number, WalStreamRunner>(this.streams.entries());
-    const replicating = await this.storage.getReplicatingSyncRules();
-    const newStreams = new Map<number, WalStreamRunner>();
-    for (let syncRules of replicating) {
-      const existing = existingStreams.get(syncRules.id);
-      if (existing && !existing.stopped) {
-        // No change
-        existingStreams.delete(syncRules.id);
-        newStreams.set(syncRules.id, existing);
-      } else if (existing && existing.stopped) {
-        // Stopped (e.g. fatal error, slot rename).
-        // Remove from the list. Next refresh call will restart the stream.
-        existingStreams.delete(syncRules.id);
-      } else {
-        // New (or resume after restart)
-        try {
-          let lock: storage.ReplicationLock;
-          if (configured_lock?.sync_rules_id == syncRules.id) {
-            lock = configured_lock;
-          } else {
-            lock = await syncRules.lock();
-          }
-          const parsed = syncRules.parsed();
-          const storage = this.storage.getInstance(parsed);
-          const stream = new WalStreamRunner({
-            factory: this.storage,
-            storage: storage,
-            source_db: this.system.config.connection!,
-            lock,
-            rateLimiter: this.rateLimiter
-          });
-          newStreams.set(syncRules.id, stream);
-          stream.start();
-        } catch (e) {
-          // Could be a sync rules parse error,
-          // for example from stricter validation that was added.
-          // This will be retried every couple of seconds.
-          // When new (valid) sync rules are deployed and processed, this one be disabled.
-          logger.error(`Failed to start replication for ${syncRules.slot_name}`, e);
-        }
-      }
-    }
-
-    this.streams = newStreams;
-
-    // TODO: Should this termination be happening in the "background" instead?
-    // That becomes tricky to manage
-
-    for (let stream of existingStreams.values()) {
-      // Old - stop and remove.
-      try {
-        await stream.terminate();
-      } catch (e) {
-        // This will be retried
-        logger.warn(`Failed to terminate ${stream.slot_name}`, e);
-      }
-    }
-
-    // Sync rules stopped previously or by a different process.
-    const stopped = await this.storage.getStoppedSyncRules();
-    for (let syncRules of stopped) {
-      try {
-        const lock = await syncRules.lock();
-        try {
-          const parsed = syncRules.parsed();
-          const storage = this.storage.getInstance(parsed);
-          const stream = new WalStreamRunner({
-            factory: this.storage,
-            storage: storage,
-            source_db: this.system.config.connection!,
-            lock
-          });
-          await stream.terminate();
-        } finally {
-          await lock.release();
-        }
-      } catch (e) {
-        logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
-      }
-    }
-  }
-}

package/src/replication/WalStreamRunner.ts

@@ -1,180 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-import * as storage from '../storage/storage-index.js';
-import * as util from '../util/util-index.js';
-
-import { ErrorRateLimiter } from './ErrorRateLimiter.js';
-import { MissingReplicationSlotError, WalStream } from './WalStream.js';
-import { ResolvedConnection } from '../util/config/types.js';
-import { container, logger } from '@powersync/lib-services-framework';
-
-export interface WalStreamRunnerOptions {
-  factory: storage.BucketStorageFactory;
-  storage: storage.SyncRulesBucketStorage;
-  source_db: ResolvedConnection;
-  lock: storage.ReplicationLock;
-  rateLimiter?: ErrorRateLimiter;
-}
-
-export class WalStreamRunner {
-  private abortController = new AbortController();
-
-  private runPromise?: Promise<void>;
-
-  private connections: util.PgManager | null = null;
-
-  private rateLimiter?: ErrorRateLimiter;
-
-  constructor(public options: WalStreamRunnerOptions) {
-    this.rateLimiter = options.rateLimiter;
-  }
-
-  start() {
-    this.runPromise = this.run();
-  }
-
-  get slot_name() {
-    return this.options.storage.slot_name;
-  }
-
-  get stopped() {
-    return this.abortController.signal.aborted;
-  }
-
-  async run() {
-    try {
-      await this.replicateLoop();
-    } catch (e) {
-      // Fatal exception
-      container.reporter.captureException(e, {
-        metadata: {
-          replication_slot: this.slot_name
-        }
-      });
-      logger.error(`Replication failed on ${this.slot_name}`, e);
-
-      if (e instanceof MissingReplicationSlotError) {
-        // This stops replication on this slot, and creates a new slot
-        await this.options.storage.factory.slotRemoved(this.slot_name);
-      }
-    } finally {
-      this.abortController.abort();
-    }
-    await this.options.lock.release();
-  }
-
-  async replicateLoop() {
-    while (!this.stopped) {
-      await this.replicateOnce();
-
-      if (!this.stopped) {
-        await new Promise((resolve) => setTimeout(resolve, 5000));
-      }
-    }
-  }
-
-  async replicateOnce() {
-    // New connections on every iteration (every error with retry),
-    // otherwise we risk repeating errors related to the connection,
-    // such as caused by cached PG schemas.
-    let connections = new util.PgManager(this.options.source_db, {
-      // Pool connections are only used intermittently.
-      idleTimeout: 30_000,
-      maxSize: 2
-    });
-    this.connections = connections;
-    try {
-      await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
-      if (this.stopped) {
-        return;
-      }
-      const stream = new WalStream({
-        abort_signal: this.abortController.signal,
-        factory: this.options.factory,
-        storage: this.options.storage,
-        connections
-      });
-      await stream.replicate();
-    } catch (e) {
-      logger.error(`Replication error`, e);
-      if (e.cause != null) {
-        // Example:
-        // PgError.conn_ended: Unable to do postgres query on ended connection
-        //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
-        //     at stream.next (<anonymous>)
-        //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
-        //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
-        //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
-        //     ...
-        //   cause: TypeError: match is not iterable
-        //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
-        //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
-        //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
-        //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
-        //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
-        //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
-        //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
-        //   [Symbol(pg.ErrorCode)]: 'conn_ended',
-        //   [Symbol(pg.ErrorResponse)]: undefined
-        // }
-        // Without this additional log, the cause would not be visible in the logs.
-        logger.error(`cause`, e.cause);
-      }
-      if (e instanceof MissingReplicationSlotError) {
-        throw e;
-      } else {
-        // Report the error if relevant, before retrying
-        container.reporter.captureException(e, {
-          metadata: {
-            replication_slot: this.slot_name
-          }
-        });
-        // This sets the retry delay
-        this.rateLimiter?.reportError(e);
-      }
-    } finally {
-      this.connections = null;
-      if (connections != null) {
-        await connections.end();
-      }
-    }
-  }
-
-  /**
-   * This will also release the lock if start() was called earlier.
-   */
-  async stop(options?: { force?: boolean }) {
-    logger.info(`${this.slot_name} Stopping replication`);
-    // End gracefully
-    this.abortController.abort();
-
-    if (options?.force) {
-      // destroy() is more forceful.
-      await this.connections?.destroy();
-    }
-    await this.runPromise;
-  }
-
-  /**
-   * Terminate this replication stream. This drops the replication slot and deletes the replication data.
-   *
-   * Stops replication if needed.
-   */
-  async terminate(options?: { force?: boolean }) {
-    logger.info(`${this.slot_name} Terminating replication`);
-    await this.stop(options);
-
-    const slotName = this.slot_name;
-    const db = await pgwire.connectPgWire(this.options.source_db, { type: 'standard' });
-    try {
-      await db.query({
-        statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
-        params: [{ type: 'varchar', value: slotName }]
-      });
-    } finally {
-      await db.end();
-    }
-
-    await this.options.storage.terminate();
-  }
-}

package/src/replication/util.ts

@@ -1,76 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-import * as util from '../util/util-index.js';
-import { ReplicationColumn, ReplicationIdentity } from './PgRelation.js';
-
-export interface ReplicaIdentityResult {
-  columns: ReplicationColumn[];
-  replicationIdentity: ReplicationIdentity;
-}
-
-export async function getPrimaryKeyColumns(
-  db: pgwire.PgClient,
-  relationId: number,
-  mode: 'primary' | 'replident'
-): Promise<ReplicationColumn[]> {
-  const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`;
-  const attrRows = await util.retriedQuery(db, {
-    statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
-                                    FROM pg_index i
-                                             JOIN pg_attribute a
-                                                  ON a.attrelid = i.indrelid
-                                                      AND a.attnum = ANY (i.indkey)
-                                    WHERE i.indrelid = $1::oid
-                                      AND ${indexFlag}
-                                      AND a.attnum > 0
-                                      ORDER BY a.attnum`,
-    params: [{ value: relationId, type: 'int4' }]
-  });
-
-  return attrRows.rows.map((row) => {
-    return { name: row[0] as string, typeOid: row[1] as number };
-  });
-}
-
-export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise<ReplicationColumn[]> {
-  const attrRows = await util.retriedQuery(db, {
-    statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
-                                    FROM pg_attribute a
-                                    WHERE a.attrelid = $1::oid
-                                      AND attnum > 0
-                                    ORDER BY a.attnum`,
-    params: [{ type: 'varchar', value: relationId }]
-  });
-  return attrRows.rows.map((row) => {
-    return { name: row[0] as string, typeOid: row[1] as number };
-  });
-}
-
-export async function getReplicationIdentityColumns(
-  db: pgwire.PgClient,
-  relationId: number
-): Promise<ReplicaIdentityResult> {
-  const rows = await util.retriedQuery(db, {
-    statement: `SELECT CASE relreplident
-        WHEN 'd' THEN 'default'
-        WHEN 'n' THEN 'nothing'
-        WHEN 'f' THEN 'full'
-        WHEN 'i' THEN 'index'
-     END AS replica_identity
-FROM pg_class
-WHERE oid = $1::oid LIMIT 1`,
-    params: [{ type: 'int8', value: relationId }]
-  });
-  const idType: string = rows.rows[0]?.[0];
-  if (idType == 'nothing' || idType == null) {
-    return { replicationIdentity: 'nothing', columns: [] };
-  } else if (idType == 'full') {
-    return { replicationIdentity: 'full', columns: await getAllColumns(db, relationId) };
-  } else if (idType == 'default') {
-    return { replicationIdentity: 'default', columns: await getPrimaryKeyColumns(db, relationId, 'primary') };
-  } else if (idType == 'index') {
-    return { replicationIdentity: 'index', columns: await getPrimaryKeyColumns(db, relationId, 'replident') };
-  } else {
-    return { replicationIdentity: 'nothing', columns: [] };
-  }
-}

package/src/system/CorePowerSyncSystem.ts

@@ -1,64 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { LifeCycledSystem, container, logger } from '@powersync/lib-services-framework';
-
-import * as storage from '../storage/storage-index.js';
-import * as utils from '../util/util-index.js';
-
-export abstract class CorePowerSyncSystem extends LifeCycledSystem {
-  abstract storage: storage.BucketStorageFactory;
-  abstract pgwire_pool?: pgwire.PgClient;
-  closed: boolean;
-
-  protected stopHandlers: Set<() => void> = new Set();
-
-  constructor(public config: utils.ResolvedPowerSyncConfig) {
-    super();
-    this.closed = false;
-  }
-
-  get client_keystore() {
-    return this.config.client_keystore;
-  }
-
-  get dev_client_keystore() {
-    return this.config.dev_client_keystore;
-  }
-
-  /**
-   * Adds a termination handler which will call handlers registered via
-   * [addStopHandler].
-   * This should be called after the server is started and it's termination handler is added.
-   * This is so that the handler is run before the server's handler, allowing streams to be interrupted on exit
-   */
-  addTerminationHandler() {
-    container.terminationHandler.handleTerminationSignal(async () => {
-      // Close open streams, so that they don't block the server from closing.
-      // Note: This does not work well when streaming requests are queued. In that case, the server still doesn't
-      // close in the 30-second timeout.
-      this.closed = true;
-      logger.info(`Closing ${this.stopHandlers.size} streams`);
-      for (let handler of this.stopHandlers) {
-        handler();
-      }
-    });
-  }
-
-  addStopHandler(handler: () => void): () => void {
-    if (this.closed) {
-      handler();
-      return () => {};
-    }
-    this.stopHandlers.add(handler);
-    return () => {
-      this.stopHandlers.delete(handler);
-    };
-  }
-
-  requirePgPool() {
-    if (this.pgwire_pool == null) {
-      throw new Error('No source connection configured');
-    } else {
-      return this.pgwire_pool!;
-    }
-  }
-}

package/src/util/PgManager.ts

@@ -1,64 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { NormalizedPostgresConnection } from '@powersync/service-types';
-
-export class PgManager {
-  /**
-   * Do not use this for any transactions.
-   */
-  public readonly pool: pgwire.PgClient;
-
-  private connectionPromises: Promise<pgwire.PgConnection>[] = [];
-
-  constructor(public options: NormalizedPostgresConnection, public poolOptions: pgwire.PgPoolOptions) {
-    // The pool is lazy - no connections are opened until a query is performed.
-    this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
-  }
-
-  /**
-   * Create a new replication connection.
-   */
-  async replicationConnection(): Promise<pgwire.PgConnection> {
-    const p = pgwire.connectPgWire(this.options, { type: 'replication' });
-    this.connectionPromises.push(p);
-    return await p;
-  }
-
-  /**
-   * Create a new standard connection, used for initial snapshot.
-   *
-   * This connection must not be shared between multiple async contexts.
-   */
-  async snapshotConnection(): Promise<pgwire.PgConnection> {
-    const p = pgwire.connectPgWire(this.options, { type: 'standard' });
-    this.connectionPromises.push(p);
-    return await p;
-  }
-
-  async end() {
-    for (let result of await Promise.allSettled([
-      this.pool.end(),
-      ...this.connectionPromises.map((promise) => {
-        return promise.then((connection) => connection.end());
-      })
-    ])) {
-      // Throw the first error, if any
-      if (result.status == 'rejected') {
-        throw result.reason;
-      }
-    }
-  }
-
-  async destroy() {
-    this.pool.destroy();
-    for (let result of await Promise.allSettled([
-      ...this.connectionPromises.map((promise) => {
-        return promise.then((connection) => connection.destroy());
-      })
-    ])) {
-      // Throw the first error, if any
-      if (result.status == 'rejected') {
-        throw result.reason;
-      }
-    }
-  }
-}

package/src/util/migration_lib.ts

@@ -1,79 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
-
-interface Migration {
-  id: number;
-  name: string;
-  up: MigrationFunction;
-}
-
-// Very loosely based on https://github.com/porsager/postgres-shift/
-export class Migrations {
-  private migrations: Migration[] = [];
-
-  add(id: number, name: string, up: MigrationFunction) {
-    if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
-      throw new Error('Migration ids must be strictly incrementing');
-    }
-    this.migrations.push({ id, up, name });
-  }
-
-  async up(db: pgwire.PgConnection) {
-    await db.query('BEGIN');
-    try {
-      await this.ensureMigrationsTable(db);
-      const current = await this.getCurrentMigration(db);
-      let currentId = current ? current.id : 0;
-
-      for (let migration of this.migrations) {
-        if (migration.id <= currentId) {
-          continue;
-        }
-        await migration.up(db);
-
-        await db.query({
-          statement: `
-      insert into migrations (
-        migration_id,
-        name
-      ) values (
-        $1,
-        $2
-      )
-    `,
-          params: [
-            { type: 'int4', value: migration.id },
-            { type: 'varchar', value: migration.name }
-          ]
-        });
-      }
-
-      await db.query('COMMIT');
-    } catch (e) {
-      await db.query('ROLLBACK');
-      throw e;
-    }
-  }
-
-  getCurrentMigration(db: pgwire.PgConnection) {
-    return db
-      .query(
-        `
-      select migration_id as id from migrations
-      order by migration_id desc
-      limit 1
-    `
-      )
-      .then((results) => ({ id: results.rows[0][0] as number }));
-  }
-
-  async ensureMigrationsTable(db: pgwire.PgConnection) {
-    await db.query(`create table if not exists migrations (
-        migration_id serial primary key,
-        created_at timestamp with time zone not null default now(),
-        name text
-      )
-    `);
-  }
-}