@powersync/service-module-postgres 0.19.2 → 0.19.4

package/src/replication/WalStreamReplicationJob.ts

@@ -1,138 +0,0 @@
-import { container, logger } from '@powersync/lib-services-framework';
-import { PgManager } from './PgManager.js';
-import { MissingReplicationSlotError, sendKeepAlive, WalStream } from './WalStream.js';
-
-import { replication } from '@powersync/service-core';
-import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
-import { getApplicationName } from '../utils/application-name.js';
-
-export interface WalStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions {
-  connectionFactory: ConnectionManagerFactory;
-}
-
-export class WalStreamReplicationJob extends replication.AbstractReplicationJob {
-  private connectionFactory: ConnectionManagerFactory;
-  private connectionManager: PgManager | null = null;
-  private lastStream: WalStream | null = null;
-
-  constructor(options: WalStreamReplicationJobOptions) {
-    super(options);
-    this.logger = logger.child({ prefix: `[${this.slotName}] ` });
-    this.connectionFactory = options.connectionFactory;
-  }
-
-  /**
-   * Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
-   *
-   * The old WAL files are only deleted once no replication slot still references it.
-   *
-   * Unfortunately, when there are no changes to the db, the database creates new WAL files without the replication slot
-   * advancing**.
-   *
-   * As a workaround, we write a new message every couple of minutes, to make sure that the replication slot advances.
-   *
-   * **This may be a bug in pgwire or how we're using it.
-   */
-  async keepAlive() {
-    if (this.connectionManager) {
-      try {
-        await sendKeepAlive(this.connectionManager.pool);
-      } catch (e) {
-        this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
-      }
-    }
-  }
-
-  get slotName() {
-    return this.options.storage.slot_name;
-  }
-
-  async replicate() {
-    try {
-      await this.replicateOnce();
-    } catch (e) {
-      // Fatal exception
-
-      if (!this.isStopped) {
-        // Ignore aborted errors
-
-        this.logger.error(`Replication error`, e);
-        if (e.cause != null) {
-          // Example:
-          // PgError.conn_ended: Unable to do postgres query on ended connection
-          //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
-          //     at stream.next (<anonymous>)
-          //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
-          //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
-          //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
-          //     ...
-          //   cause: TypeError: match is not iterable
-          //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
-          //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
-          //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
-          //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
-          //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
-          //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
-          //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
-          //   [Symbol(pg.ErrorCode)]: 'conn_ended',
-          //   [Symbol(pg.ErrorResponse)]: undefined
-          // }
-          // Without this additional log, the cause would not be visible in the logs.
-          this.logger.error(`cause`, e.cause);
-        }
-        // Report the error if relevant, before retrying
-        container.reporter.captureException(e, {
-          metadata: {
-            replication_slot: this.slotName
-          }
-        });
-        // This sets the retry delay
-        this.rateLimiter.reportError(e);
-      }
-
-      if (e instanceof MissingReplicationSlotError) {
-        // This stops replication on this slot and restarts with a new slot
-        await this.options.storage.factory.restartReplication(this.storage.group_id);
-      }
-
-      // No need to rethrow - the error is already logged, and retry behavior is the same on error
-    } finally {
-      this.abortController.abort();
-    }
-  }
-
-  async replicateOnce() {
-    // New connections on every iteration (every error with retry),
-    // otherwise we risk repeating errors related to the connection,
-    // such as caused by cached PG schemas.
-    const connectionManager = this.connectionFactory.create({
-      // Pool connections are only used intermittently.
-      idleTimeout: 30_000,
-      maxSize: 2,
-      applicationName: getApplicationName()
-    });
-    this.connectionManager = connectionManager;
-    try {
-      await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
-      if (this.isStopped) {
-        return;
-      }
-      const stream = new WalStream({
-        logger: this.logger,
-        abort_signal: this.abortController.signal,
-        storage: this.options.storage,
-        metrics: this.options.metrics,
-        connections: connectionManager
-      });
-      this.lastStream = stream;
-      await stream.replicate();
-    } finally {
-      this.connectionManager = null;
-      await connectionManager.end();
-    }
-  }
-
-  async getReplicationLagMillis(): Promise<number | undefined> {
-    return this.lastStream?.getReplicationLagMillis();
-  }
-}

package/src/replication/WalStreamReplicator.ts

@@ -1,79 +0,0 @@
-import { replication, storage } from '@powersync/service-core';
-import { PostgresModule } from '../module/PostgresModule.js';
-import { getApplicationName } from '../utils/application-name.js';
-import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
-import { cleanUpReplicationSlot } from './replication-utils.js';
-import { WalStreamReplicationJob } from './WalStreamReplicationJob.js';
-
-export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
-  connectionFactory: ConnectionManagerFactory;
-}
-
-export class WalStreamReplicator extends replication.AbstractReplicator<WalStreamReplicationJob> {
-  private readonly connectionFactory: ConnectionManagerFactory;
-
-  constructor(options: WalStreamReplicatorOptions) {
-    super(options);
-    this.connectionFactory = options.connectionFactory;
-  }
-
-  createJob(options: replication.CreateJobOptions): WalStreamReplicationJob {
-    return new WalStreamReplicationJob({
-      id: this.createJobId(options.storage.group_id),
-      storage: options.storage,
-      metrics: this.metrics,
-      connectionFactory: this.connectionFactory,
-      lock: options.lock,
-      rateLimiter: this.rateLimiter
-    });
-  }
-
-  async cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise<void> {
-    const connectionManager = this.connectionFactory.create({
-      applicationName: getApplicationName(),
-      idleTimeout: 30_000,
-      maxSize: 1
-    });
-    try {
-      // TODO: Slot_name will likely have to come from a different source in the future
-      await cleanUpReplicationSlot(syncRulesStorage.slot_name, connectionManager.pool);
-    } finally {
-      await connectionManager.end();
-    }
-  }
-
-  async stop(): Promise<void> {
-    await super.stop();
-    await this.connectionFactory.shutdown();
-  }
-
-  async testConnection() {
-    return await PostgresModule.testConnection(this.connectionFactory.dbConnectionConfig);
-  }
-
-  async getReplicationLagMillis(): Promise<number | undefined> {
-    const lag = await super.getReplicationLagMillis();
-    if (lag != null) {
-      return lag;
-    }
-
-    // Booting or in an error loop. Check last active replication status.
-    // This includes sync rules in an ERROR state.
-    const content = await this.storage.getActiveSyncRulesContent();
-    if (content == null) {
-      return undefined;
-    }
-    // Measure the lag from the last commit or keepalive timestamp.
-    // This is not 100% accurate since it is the commit time in the storage db rather than
-    // the source db, but it's the best we have for postgres.
-
-    const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0;
-    const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0;
-    const latestTs = Math.max(checkpointTs, keepaliveTs);
-    if (latestTs != 0) {
-      return Date.now() - latestTs;
-    }
-
-    return undefined;
-  }
-}

package/src/replication/replication-index.ts

@@ -1,5 +0,0 @@
-export * from './PgRelation.js';
-export * from './replication-utils.js';
-export * from './WalStream.js';
-export * from './WalStreamReplicator.js';
-export * from './WalStreamReplicationJob.js';

package/src/replication/replication-utils.ts

@@ -1,398 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-
-import * as lib_postgres from '@powersync/lib-service-postgres';
-import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
-import { PatternResult, storage } from '@powersync/service-core';
-import * as sync_rules from '@powersync/service-sync-rules';
-import * as service_types from '@powersync/service-types';
-import { ReplicationIdentity } from './PgRelation.js';
-
-export interface ReplicaIdentityResult {
-  replicationColumns: storage.ColumnDescriptor[];
-  replicationIdentity: ReplicationIdentity;
-}
-
-export async function getPrimaryKeyColumns(
-  db: pgwire.PgClient,
-  relationId: number,
-  mode: 'primary' | 'replident'
-): Promise<storage.ColumnDescriptor[]> {
-  const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`;
-  const attrRows = await lib_postgres.retriedQuery(db, {
-    statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum
-                                    FROM pg_index i
-                                    JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey)
-                                    JOIN pg_type t ON a.atttypid = t.oid
-                                    WHERE i.indrelid = $1::oid
-                                      AND ${indexFlag}
-                                      AND a.attnum > 0
-                                      ORDER BY a.attnum`,
-    params: [{ value: relationId, type: 'int4' }]
-  });
-
-  return attrRows.rows.map((row) => {
-    return {
-      name: row.decodeWithoutCustomTypes(0) as string,
-      typeId: row.decodeWithoutCustomTypes(1) as number
-    } satisfies storage.ColumnDescriptor;
-  });
-}
-
-export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise<storage.ColumnDescriptor[]> {
-  const attrRows = await lib_postgres.retriedQuery(db, {
-    statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum
-                                    FROM pg_attribute a
-                                    JOIN pg_type t ON a.atttypid = t.oid
-                                    WHERE a.attrelid = $1::oid
-                                      AND attnum > 0
-                                    ORDER BY a.attnum`,
-    params: [{ type: 'varchar', value: relationId }]
-  });
-  return attrRows.rows.map((row) => {
-    return {
-      name: row.decodeWithoutCustomTypes(0) as string,
-      typeId: row.decodeWithoutCustomTypes(1) as number
-    } satisfies storage.ColumnDescriptor;
-  });
-}
-
-export async function getReplicationIdentityColumns(
-  db: pgwire.PgClient,
-  relationId: number
-): Promise<ReplicaIdentityResult> {
-  const rows = await lib_postgres.retriedQuery(db, {
-    statement: `SELECT CASE relreplident
-        WHEN 'd' THEN 'default'
-        WHEN 'n' THEN 'nothing'
-        WHEN 'f' THEN 'full'
-        WHEN 'i' THEN 'index'
-     END AS replica_identity
-FROM pg_class
-WHERE oid = $1::oid LIMIT 1`,
-    params: [{ type: 'int8', value: relationId }]
-  });
-  const idType: string = rows.rows[0]?.decodeWithoutCustomTypes(0);
-  if (idType == 'nothing' || idType == null) {
-    return { replicationIdentity: 'nothing', replicationColumns: [] };
-  } else if (idType == 'full') {
-    return { replicationIdentity: 'full', replicationColumns: await getAllColumns(db, relationId) };
-  } else if (idType == 'default') {
-    return {
-      replicationIdentity: 'default',
-      replicationColumns: await getPrimaryKeyColumns(db, relationId, 'primary')
-    };
-  } else if (idType == 'index') {
-    return {
-      replicationIdentity: 'index',
-      replicationColumns: await getPrimaryKeyColumns(db, relationId, 'replident')
-    };
-  } else {
-    return { replicationIdentity: 'nothing', replicationColumns: [] };
-  }
-}
-
-export async function checkSourceConfiguration(db: pgwire.PgClient, publicationName: string): Promise<void> {
-  // Check basic config
-  await lib_postgres.retriedQuery(
-    db,
-    `DO $$
-BEGIN
-if current_setting('wal_level') is distinct from 'logical' then
-raise exception 'wal_level must be set to ''logical'', your database has it set to ''%''. Please edit your config file and restart PostgreSQL.', current_setting('wal_level');
-end if;
-if (current_setting('max_replication_slots')::int >= 1) is not true then
-raise exception 'Your max_replication_slots setting is too low, it must be greater than 1. Please edit your config file and restart PostgreSQL.';
-end if;
-if (current_setting('max_wal_senders')::int >= 1) is not true then
-raise exception 'Your max_wal_senders setting is too low, it must be greater than 1. Please edit your config file and restart PostgreSQL.';
-end if;
-end;
-$$ LANGUAGE plpgsql;`
-  );
-
-  // Check that publication exists
-  const rs = await lib_postgres.retriedQuery(db, {
-    statement: `SELECT * FROM pg_publication WHERE pubname = $1`,
-    params: [{ type: 'varchar', value: publicationName }]
-  });
-  const row = pgwire.pgwireRows(rs)[0];
-  if (row == null) {
-    throw new ServiceError(
-      ErrorCode.PSYNC_S1141,
-      `Publication '${publicationName}' does not exist. Run: \`CREATE PUBLICATION ${publicationName} FOR ALL TABLES\`, or read the documentation for details.`
-    );
-  }
-  if (row.pubinsert == false || row.pubupdate == false || row.pubdelete == false || row.pubtruncate == false) {
-    throw new ServiceError(
-      ErrorCode.PSYNC_S1142,
-      `Publication '${publicationName}' does not publish all changes. Create a publication using \`WITH (publish = "insert, update, delete, truncate")\` (the default).`
-    );
-  }
-  if (row.pubviaroot) {
-    throw new ServiceError(
-      ErrorCode.PSYNC_S1143,
-      `'${publicationName}' uses publish_via_partition_root, which is not supported.`
-    );
-  }
-}
-
-export async function checkTableRls(
-  db: pgwire.PgClient,
-  relationId: number
-): Promise<{ canRead: boolean; message?: string }> {
-  const rs = await lib_postgres.retriedQuery(db, {
-    statement: `
-WITH user_info AS (
-    SELECT 
-        current_user as username,
-        r.rolsuper,
-        r.rolbypassrls
-    FROM pg_roles r 
-    WHERE r.rolname = current_user
-)
-SELECT
-    c.relname as tablename,
-    c.relrowsecurity as rls_enabled,
-    u.username as username,
-    u.rolsuper as is_superuser,
-    u.rolbypassrls as bypasses_rls
-FROM pg_class c
-CROSS JOIN user_info u
-WHERE c.oid = $1::oid;
-`,
-    params: [{ type: 'int4', value: relationId }]
-  });
-
-  const rows = pgwire.pgwireRows<{
-    rls_enabled: boolean;
-    tablename: string;
-    username: string;
-    is_superuser: boolean;
-    bypasses_rls: boolean;
-  }>(rs);
-  if (rows.length == 0) {
-    // Not expected, since we already got the oid
-    throw new ServiceAssertionError(`Table with OID ${relationId} does not exist.`);
-  }
-  const row = rows[0];
-  if (row.is_superuser || row.bypasses_rls) {
-    // Bypasses RLS automatically.
-    return { canRead: true };
-  }
-
-  if (row.rls_enabled) {
-    // Don't skip, since we _may_ still be able to get results.
-    return {
-      canRead: false,
-      message: `[${ErrorCode.PSYNC_S1145}] Row Level Security is enabled on table "${row.tablename}". To make sure that ${row.username} can read the table, run: 'ALTER ROLE ${row.username} BYPASSRLS'.`
-    };
-  }
-
-  return { canRead: true };
-}
-
-export interface GetDebugTablesInfoOptions {
-  db: pgwire.PgClient;
-  publicationName: string;
-  connectionTag: string;
-  tablePatterns: sync_rules.TablePattern[];
-  syncRules: sync_rules.SyncConfig;
-}
-
-export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Promise<PatternResult[]> {
-  const { db, publicationName, connectionTag, tablePatterns, syncRules } = options;
-  let result: PatternResult[] = [];
-
-  for (let tablePattern of tablePatterns) {
-    const schema = tablePattern.schema;
-
-    let patternResult: PatternResult = {
-      schema: schema,
-      pattern: tablePattern.tablePattern,
-      wildcard: tablePattern.isWildcard
-    };
-    result.push(patternResult);
-
-    if (tablePattern.isWildcard) {
-      patternResult.tables = [];
-      const prefix = tablePattern.tablePrefix;
-      const results = await lib_postgres.retriedQuery(db, {
-        statement: `SELECT c.oid AS relid, c.relname AS table_name
-        FROM pg_class c
-        JOIN pg_namespace n ON n.oid = c.relnamespace
-        WHERE n.nspname = $1
-        AND c.relkind = 'r'
-        AND c.relname LIKE $2`,
-        params: [
-          { type: 'varchar', value: schema },
-          { type: 'varchar', value: tablePattern.tablePattern }
-        ]
-      });
-
-      for (let row of pgwire.pgwireRows(results)) {
-        const name = row.table_name as string;
-        const relationId = row.relid as number;
-        if (!name.startsWith(prefix)) {
-          continue;
-        }
-        const details = await getDebugTableInfo({
-          db,
-          name,
-          publicationName,
-          connectionTag,
-          tablePattern,
-          relationId,
-          syncRules: syncRules
-        });
-        patternResult.tables.push(details);
-      }
-    } else {
-      const results = await lib_postgres.retriedQuery(db, {
-        statement: `SELECT c.oid AS relid, c.relname AS table_name
-        FROM pg_class c
-        JOIN pg_namespace n ON n.oid = c.relnamespace
-        WHERE n.nspname = $1
-        AND c.relkind = 'r'
-        AND c.relname = $2`,
-        params: [
-          { type: 'varchar', value: schema },
-          { type: 'varchar', value: tablePattern.tablePattern }
-        ]
-      });
-      if (results.rows.length == 0) {
-        // Table not found
-        patternResult.table = await getDebugTableInfo({
-          db,
-          name: tablePattern.name,
-          publicationName,
-          connectionTag,
-          tablePattern,
-          relationId: null,
-          syncRules: syncRules
-        });
-      } else {
-        const row = pgwire.pgwireRows(results)[0];
-        const name = row.table_name as string;
-        const relationId = row.relid as number;
-        patternResult.table = await getDebugTableInfo({
-          db,
-          name,
-          publicationName,
-          connectionTag,
-          tablePattern,
-          relationId,
-          syncRules: syncRules
-        });
-      }
-    }
-  }
-  return result;
-}
-
-export interface GetDebugTableInfoOptions {
-  db: pgwire.PgClient;
-  name: string;
-  publicationName: string;
-  connectionTag: string;
-  tablePattern: sync_rules.TablePattern;
-  relationId: number | null;
-  syncRules: sync_rules.SyncConfig;
-}
-
-export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Promise<service_types.TableInfo> {
-  const { db, name, publicationName, connectionTag, tablePattern, relationId, syncRules } = options;
-  const schema = tablePattern.schema;
-  let id_columns_result: ReplicaIdentityResult | undefined = undefined;
-  let id_columns_error = null;
-
-  if (relationId != null) {
-    try {
-      id_columns_result = await getReplicationIdentityColumns(db, relationId);
-    } catch (e) {
-      id_columns_error = { level: 'fatal', message: e.message };
-    }
-  }
-
-  const id_columns = id_columns_result?.replicationColumns ?? [];
-
-  const sourceTable = new storage.SourceTable({
-    id: '', // not used
-    connectionTag: connectionTag,
-    objectId: relationId ?? 0,
-    schema: schema,
-    name: name,
-    replicaIdColumns: id_columns,
-    snapshotComplete: true
-  });
-
-  const syncData = syncRules.tableSyncsData(sourceTable);
-  const syncParameters = syncRules.tableSyncsParameters(sourceTable);
-
-  if (relationId == null) {
-    return {
-      schema: schema,
-      name: name,
-      pattern: tablePattern.isWildcard ? tablePattern.tablePattern : undefined,
-      replication_id: [],
-      data_queries: syncData,
-      parameter_queries: syncParameters,
-      // Also
-      errors: [{ level: 'warning', message: `Table ${sourceTable.qualifiedName} not found.` }]
-    };
-  }
-  if (id_columns.length == 0 && id_columns_error == null) {
-    let message = `No replication id found for ${sourceTable.qualifiedName}. Replica identity: ${id_columns_result?.replicationIdentity}.`;
-    if (id_columns_result?.replicationIdentity == 'default') {
-      message += ' Configure a primary key on the table.';
-    }
-    id_columns_error = { level: 'fatal', message };
-  }
-
-  let selectError = null;
-  try {
-    await lib_postgres.retriedQuery(db, `SELECT * FROM ${sourceTable.qualifiedName} LIMIT 1`);
-  } catch (e) {
-    selectError = { level: 'fatal', message: e.message };
-  }
-
-  let replicateError = null;
-
-  const publications = await lib_postgres.retriedQuery(db, {
-    statement: `SELECT tablename FROM pg_publication_tables WHERE pubname = $1 AND schemaname = $2 AND tablename = $3`,
-    params: [
-      { type: 'varchar', value: publicationName },
-      { type: 'varchar', value: tablePattern.schema },
-      { type: 'varchar', value: name }
-    ]
-  });
-  if (publications.rows.length == 0) {
-    replicateError = {
-      level: 'fatal',
-      message: `Table ${sourceTable.qualifiedName} is not part of publication '${publicationName}'. Run: \`ALTER PUBLICATION ${publicationName} ADD TABLE ${sourceTable.qualifiedName}\`.`
-    };
-  }
-
-  const rlsCheck = await checkTableRls(db, relationId);
-  const rlsError = rlsCheck.canRead ? null : { message: rlsCheck.message!, level: 'warning' };
-
-  return {
-    schema: schema,
-    name: name,
-    pattern: tablePattern.isWildcard ? tablePattern.tablePattern : undefined,
-    replication_id: id_columns.map((c) => c.name),
-    data_queries: syncData,
-    parameter_queries: syncParameters,
-    errors: [id_columns_error, selectError, replicateError, rlsError].filter(
-      (error) => error != null
-    ) as service_types.ReplicationError[]
-  };
-}
-
-export async function cleanUpReplicationSlot(slotName: string, db: pgwire.PgClient): Promise<void> {
-  logger.info(`Cleaning up Postgres replication slot: ${slotName}...`);
-
-  await db.query({
-    statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
-    params: [{ type: 'varchar', value: slotName }]
-  });
-}