@powersync/service-module-postgres 0.19.2 → 0.19.4

package/src/api/PostgresRouteAPIAdapter.ts

@@ -1,356 +0,0 @@
-import * as lib_postgres from '@powersync/lib-service-postgres';
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
-import { api, ParseSyncRulesOptions, ReplicationHeadCallback } from '@powersync/service-core';
-import * as pgwire from '@powersync/service-jpgwire';
-import * as sync_rules from '@powersync/service-sync-rules';
-import * as service_types from '@powersync/service-types';
-import * as replication_utils from '../replication/replication-utils.js';
-import { getDebugTableInfo } from '../replication/replication-utils.js';
-import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js';
-import * as types from '../types/types.js';
-import { getApplicationName } from '../utils/application-name.js';
-import { CustomTypeRegistry } from '../types/registry.js';
-import { PostgresTypeResolver } from '../types/resolver.js';
-
-export class PostgresRouteAPIAdapter implements api.RouteAPI {
-  private typeCache: PostgresTypeResolver;
-  connectionTag: string;
-  // TODO this should probably be configurable one day
-  publicationName = PUBLICATION_NAME;
-
-  static withConfig(config: types.ResolvedConnectionConfig) {
-    const pool = pgwire.connectPgWirePool(config, {
-      idleTimeout: 30_000,
-      applicationName: getApplicationName()
-    });
-    return new PostgresRouteAPIAdapter(pool, config.tag, config);
-  }
-
-  /**
-   * @param config - Required for the service; optional for tests.
-   */
-  constructor(
-    protected pool: pgwire.PgClient,
-    connectionTag?: string,
-    private config?: types.ResolvedConnectionConfig
-  ) {
-    this.typeCache = new PostgresTypeResolver(pool);
-    this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG;
-  }
-
-  getParseSyncRulesOptions(): ParseSyncRulesOptions {
-    return {
-      defaultSchema: 'public'
-    };
-  }
-
-  async shutdown(): Promise<void> {
-    await this.pool.end();
-  }
-
-  async getSourceConfig(): Promise<service_types.configFile.ResolvedDataSourceConfig> {
-    return this.config!;
-  }
-
-  async getConnectionStatus(): Promise<service_types.ConnectionStatusV2> {
-    const base = {
-      id: this.config?.id ?? '',
-      uri: this.config == null ? '' : types.baseUri(this.config)
-    };
-
-    try {
-      await lib_postgres.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`);
-    } catch (e) {
-      return {
-        ...base,
-        connected: false,
-        errors: [{ level: 'fatal', message: e.message }]
-      };
-    }
-
-    try {
-      await replication_utils.checkSourceConfiguration(this.pool, this.publicationName);
-    } catch (e) {
-      return {
-        ...base,
-        connected: true,
-        errors: [{ level: 'fatal', message: e.message }]
-      };
-    }
-
-    return {
-      ...base,
-      connected: true,
-      errors: []
-    };
-  }
-
-  async executeQuery(query: string, params: any[]): Promise<service_types.internal_routes.ExecuteSqlResponse> {
-    if (!this.config?.debug_api) {
-      return service_types.internal_routes.ExecuteSqlResponse.encode({
-        results: {
-          columns: [],
-          rows: []
-        },
-        success: false,
-        error: 'SQL querying is not enabled'
-      });
-    }
-
-    try {
-      const result = await this.pool.query({
-        statement: query,
-        params: params.map(lib_postgres.autoParameter)
-      });
-
-      return service_types.internal_routes.ExecuteSqlResponse.encode({
-        success: true,
-        results: {
-          columns: result.columns.map((c) => c.name),
-          rows: result.rows.map((row) => {
-            return row.raw.map((raw, i) => {
-              const value = pgwire.PgType.decode(raw, row.columns[i].typeOid);
-              const sqlValue = sync_rules.applyValueContext(
-                sync_rules.toSyncRulesValue(value),
-                sync_rules.CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY
-              );
-              if (typeof sqlValue == 'bigint') {
-                return Number(value);
-              } else if (sync_rules.isJsonValue(sqlValue)) {
-                return sqlValue;
-              } else {
-                return null;
-              }
-            });
-          })
-        }
-      });
-    } catch (e) {
-      return service_types.internal_routes.ExecuteSqlResponse.encode({
-        results: {
-          columns: [],
-          rows: []
-        },
-        success: false,
-        error: e.message
-      });
-    }
-  }
-
-  async getDebugTablesInfo(
-    tablePatterns: sync_rules.TablePattern[],
-    sqlSyncRules: sync_rules.SqlSyncRules
-  ): Promise<api.PatternResult[]> {
-    let result: api.PatternResult[] = [];
-
-    for (let tablePattern of tablePatterns) {
-      const schema = tablePattern.schema;
-
-      let patternResult: api.PatternResult = {
-        schema: schema,
-        pattern: tablePattern.tablePattern,
-        wildcard: tablePattern.isWildcard
-      };
-      result.push(patternResult);
-
-      if (tablePattern.isWildcard) {
-        patternResult.tables = [];
-        const prefix = tablePattern.tablePrefix;
-        const results = await lib_postgres.retriedQuery(this.pool, {
-          statement: `SELECT c.oid AS relid, c.relname AS table_name
-        FROM pg_class c
-        JOIN pg_namespace n ON n.oid = c.relnamespace
-        WHERE n.nspname = $1
-        AND c.relkind = 'r'
-        AND c.relname LIKE $2`,
-          params: [
-            { type: 'varchar', value: schema },
-            { type: 'varchar', value: tablePattern.tablePattern }
-          ]
-        });
-
-        for (let row of pgwire.pgwireRows(results)) {
-          const name = row.table_name as string;
-          const relationId = row.relid as number;
-          if (!name.startsWith(prefix)) {
-            continue;
-          }
-          const details = await this.getDebugTableInfo(tablePattern, name, relationId, sqlSyncRules);
-          patternResult.tables.push(details);
-        }
-      } else {
-        const results = await lib_postgres.retriedQuery(this.pool, {
-          statement: `SELECT c.oid AS relid, c.relname AS table_name
-        FROM pg_class c
-        JOIN pg_namespace n ON n.oid = c.relnamespace
-        WHERE n.nspname = $1
-        AND c.relkind = 'r'
-        AND c.relname = $2`,
-          params: [
-            { type: 'varchar', value: schema },
-            { type: 'varchar', value: tablePattern.tablePattern }
-          ]
-        });
-        if (results.rows.length == 0) {
-          // Table not found
-          patternResult.table = await this.getDebugTableInfo(tablePattern, tablePattern.name, null, sqlSyncRules);
-        } else {
-          const row = pgwire.pgwireRows(results)[0];
-          const name = row.table_name as string;
-          const relationId = row.relid as number;
-          patternResult.table = await this.getDebugTableInfo(tablePattern, name, relationId, sqlSyncRules);
-        }
-      }
-    }
-    return result;
-  }
-
-  protected async getDebugTableInfo(
-    tablePattern: sync_rules.TablePattern,
-    name: string,
-    relationId: number | null,
-    syncRules: sync_rules.SqlSyncRules
-  ): Promise<service_types.TableInfo> {
-    return getDebugTableInfo({
-      db: this.pool,
-      name: name,
-      publicationName: this.publicationName,
-      connectionTag: this.connectionTag,
-      tablePattern: tablePattern,
-      relationId: relationId,
-      syncRules: syncRules
-    });
-  }
-
-  async getReplicationLagBytes(options: api.ReplicationLagOptions): Promise<number | undefined> {
-    const { bucketStorage } = options;
-    const slotName = bucketStorage.slot_name;
-    const results = await lib_postgres.retriedQuery(this.pool, {
-      statement: `SELECT
-  slot_name,
-  confirmed_flush_lsn, 
-  pg_current_wal_lsn(), 
-  (pg_current_wal_lsn() - confirmed_flush_lsn) AS lsn_distance
-FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
-      params: [{ type: 'varchar', value: slotName }]
-    });
-    const [row] = pgwire.pgwireRows(results);
-    if (row) {
-      return Number(row.lsn_distance);
-    }
-
-    throw new ServiceError({
-      status: 500,
-      code: ErrorCode.PSYNC_S4001,
-      description: `Could not determine replication lag for slot ${slotName}`
-    });
-  }
-
-  async getReplicationHead(): Promise<string> {
-    // On most Postgres versions, pg_logical_emit_message() returns the correct LSN.
-    // However, on Aurora (Postgres compatible), it can return an entirely different LSN,
-    // causing the write checkpoints to never be replicated back to the client.
-    // For those, we need to use pg_current_wal_lsn() instead.
-    const { results } = await lib_postgres.retriedQuery(this.pool, `SELECT pg_current_wal_lsn() as lsn`);
-
-    const lsn = results[0].rows[0].decodeWithoutCustomTypes(0);
-    return String(lsn);
-  }
-
-  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
-    const currentLsn = await this.getReplicationHead();
-
-    const r = await callback(currentLsn);
-
-    // Note: This may not reliably trigger a new replication message on Postgres 11 or 12,
-    // in which case there could be a delay in the client receiving the write checkpoint acknowledgement.
-    // Postgres 12 already reached EOL, and this is not a critical issue, so we're not fixing it.
-    // On postgres 13+, this works reliably.
-    await lib_postgres.retriedQuery(this.pool, KEEPALIVE_STATEMENT);
-
-    return r;
-  }
-
-  async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
-    // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
-    const results = await lib_postgres.retriedQuery(
-      this.pool,
-      `SELECT
-tbl.schemaname,
-tbl.tablename,
-tbl.quoted_name,
-json_agg(a ORDER BY attnum) as columns
-FROM
-(
-  SELECT
-    n.nspname as schemaname,
-    c.relname as tablename,
-    (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) as quoted_name
-  FROM
-    pg_catalog.pg_class c
-    JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-  WHERE
-    c.relkind = 'r'
-    AND n.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
-    AND n.nspname not like 'pg_temp_%'
-    AND n.nspname not like 'pg_toast_temp_%'
-    AND c.relnatts > 0
-    AND has_schema_privilege(n.oid, 'USAGE') = true
-    AND has_table_privilege(quote_ident(n.nspname) || '.' || quote_ident(c.relname), 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') = true
-) as tbl
-LEFT JOIN (
-  SELECT
-    attrelid,
-    attname,
-    atttypid,
-    format_type(atttypid, atttypmod) as data_type,
-    (SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
-    attnum,
-    attisdropped
-  FROM
-    pg_attribute
-) as a ON (
-  a.attrelid = tbl.quoted_name::regclass
-  AND a.attnum > 0
-  AND NOT a.attisdropped
-  AND has_column_privilege(tbl.quoted_name, a.attname, 'SELECT, INSERT, UPDATE, REFERENCES')
-)
-GROUP BY schemaname, tablename, quoted_name`
-    );
-    await this.typeCache.fetchTypesForSchema();
-    const rows = pgwire.pgwireRows(results);
-
-    let schemas: Record<string, service_types.DatabaseSchema> = {};
-
-    for (let row of rows) {
-      const schema = (schemas[row.schemaname] ??= {
-        name: row.schemaname,
-        tables: []
-      });
-      const table: service_types.TableSchema = {
-        name: row.tablename,
-        columns: [] as any[]
-      };
-      schema.tables.push(table);
-
-      const columnInfo = JSON.parse(row.columns);
-      for (let column of columnInfo) {
-        let pg_type = column.pg_type as string;
-        if (pg_type.startsWith('_')) {
-          pg_type = `${pg_type.substring(1)}[]`;
-        }
-
-        const knownType = this.typeCache.registry.lookupType(Number(column.atttypid));
-        table.columns.push({
-          name: column.attname,
-          sqlite_type: sync_rules.ExpressionType.fromTypeText(knownType.sqliteType()).typeFlags,
-          type: column.data_type,
-          internal_type: column.data_type,
-          pg_type: pg_type
-        });
-      }
-    }
-
-    return Object.values(schemas);
-  }
-}

package/src/index.ts

@@ -1 +0,0 @@
-export * from './module/PostgresModule.js';

package/src/module/PostgresModule.ts

@@ -1,122 +0,0 @@
-import { baseUri, NormalizedBasePostgresConnectionConfig } from '@powersync/lib-service-postgres';
-import {
-  api,
-  ConfigurationFileSyncRulesProvider,
-  ConnectionTestResult,
-  modules,
-  replication,
-  system
-} from '@powersync/service-core';
-import * as jpgwire from '@powersync/service-jpgwire';
-import { ReplicationMetric } from '@powersync/service-types';
-import { PostgresRouteAPIAdapter } from '../api/PostgresRouteAPIAdapter.js';
-import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js';
-import { PgManager } from '../replication/PgManager.js';
-import { PostgresErrorRateLimiter } from '../replication/PostgresErrorRateLimiter.js';
-import { checkSourceConfiguration, cleanUpReplicationSlot } from '../replication/replication-utils.js';
-import { PUBLICATION_NAME } from '../replication/WalStream.js';
-import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
-import * as types from '../types/types.js';
-import { PostgresConnectionConfig } from '../types/types.js';
-import { getApplicationName } from '../utils/application-name.js';
-import { CustomTypeRegistry } from '../types/registry.js';
-
-export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
-  constructor() {
-    super({
-      name: 'Postgres',
-      type: types.POSTGRES_CONNECTION_TYPE,
-      configSchema: types.PostgresConnectionConfig
-    });
-  }
-
-  async onInitialized(context: system.ServiceContextContainer): Promise<void> {
-    // Record replicated bytes using global jpgwire metrics. Only registered if this module is replicating
-    if (context.replicationEngine) {
-      jpgwire.setMetricsRecorder({
-        addBytesRead(bytes) {
-          context.metricsEngine.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES).add(bytes);
-        }
-      });
-      this.logger.info('Successfully set up connection metrics recorder for PostgresModule.');
-    }
-  }
-
-  protected createRouteAPIAdapter(): api.RouteAPI {
-    return PostgresRouteAPIAdapter.withConfig(this.resolveConfig(this.decodedConfig!));
-  }
-
-  protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator {
-    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
-    const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
-    const connectionFactory = new ConnectionManagerFactory(normalisedConfig);
-
-    return new WalStreamReplicator({
-      id: this.getDefaultId(normalisedConfig.database),
-      syncRuleProvider: syncRuleProvider,
-      storageEngine: context.storageEngine,
-      metricsEngine: context.metricsEngine,
-      connectionFactory: connectionFactory,
-      rateLimiter: new PostgresErrorRateLimiter()
-    });
-  }
-
-  /**
-   * Combines base config with normalized connection settings
-   */
-  private resolveConfig(config: types.PostgresConnectionConfig): types.ResolvedConnectionConfig {
-    return {
-      ...config,
-      ...types.normalizeConnectionConfig(config)
-    };
-  }
-
-  async teardown(options: modules.TearDownOptions): Promise<void> {
-    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
-    const connectionManager = new PgManager(normalisedConfig, {
-      idleTimeout: 30_000,
-      maxSize: 1,
-      applicationName: getApplicationName()
-    });
-
-    try {
-      if (options.syncRules) {
-        // TODO: In the future, once we have more replication types, we will need to check if these syncRules are for Postgres
-        for (let syncRules of options.syncRules) {
-          try {
-            await cleanUpReplicationSlot(syncRules.slot_name, connectionManager.pool);
-          } catch (e) {
-            // Not really much we can do here for failures, most likely the database is no longer accessible
-            this.logger.warn(`Failed to fully clean up Postgres replication slot: ${syncRules.slot_name}`, e);
-          }
-        }
-      }
-    } finally {
-      await connectionManager.end();
-    }
-  }
-
-  async testConnection(config: PostgresConnectionConfig): Promise<ConnectionTestResult> {
-    this.decodeConfig(config);
-    const normalizedConfig = this.resolveConfig(this.decodedConfig!);
-    return await this.testConnection(normalizedConfig);
-  }
-
-  static async testConnection(normalizedConfig: NormalizedBasePostgresConnectionConfig): Promise<ConnectionTestResult> {
-    // FIXME: This is not a complete implementation yet.
-    const connectionManager = new PgManager(normalizedConfig, {
-      idleTimeout: 30_000,
-      maxSize: 1,
-      applicationName: getApplicationName()
-    });
-    const connection = await connectionManager.snapshotConnection();
-    try {
-      await checkSourceConfiguration(connection, PUBLICATION_NAME);
-    } finally {
-      await connectionManager.end();
-    }
-    return {
-      connectionDescription: baseUri(normalizedConfig)
-    };
-  }
-}

package/src/replication/ConnectionManagerFactory.ts

@@ -1,33 +0,0 @@
-import { logger } from '@powersync/lib-services-framework';
-import { PgPoolOptions } from '@powersync/service-jpgwire';
-import { NormalizedPostgresConnectionConfig } from '../types/types.js';
-import { PgManager } from './PgManager.js';
-
-export class ConnectionManagerFactory {
-  private readonly connectionManagers = new Set<PgManager>();
-  public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
-
-  constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig) {
-    this.dbConnectionConfig = dbConnectionConfig;
-  }
-
-  create(poolOptions: PgPoolOptions) {
-    const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions });
-    this.connectionManagers.add(manager);
-
-    manager.registerListener({
-      onEnded: () => {
-        this.connectionManagers.delete(manager);
-      }
-    });
-    return manager;
-  }
-
-  async shutdown() {
-    logger.info('Shutting down Postgres connection Managers...');
-    for (const manager of [...this.connectionManagers]) {
-      await manager.end();
-    }
-    logger.info('Postgres connection Managers shutdown completed.');
-  }
-}

package/src/replication/PgManager.ts

@@ -1,122 +0,0 @@
-import { BaseObserver } from '@powersync/lib-services-framework';
-import * as pgwire from '@powersync/service-jpgwire';
-import semver from 'semver';
-import { PostgresTypeResolver } from '../types/resolver.js';
-import { NormalizedPostgresConnectionConfig } from '../types/types.js';
-import { getApplicationName } from '../utils/application-name.js';
-import { getServerVersion } from '../utils/postgres_version.js';
-
-export interface PgManagerOptions extends pgwire.PgPoolOptions {}
-
-/**
- * Shorter timeout for snapshot connections than for replication connections.
- */
-const SNAPSHOT_SOCKET_TIMEOUT = 30_000;
-
-export interface PgManagerListener {
-  onEnded(): void;
-}
-
-export class PgManager extends BaseObserver<PgManagerListener> {
-  /**
-   * Do not use this for any transactions.
-   */
-  public readonly pool: pgwire.PgClient;
-
-  public readonly types: PostgresTypeResolver;
-
-  private connectionPromises: Promise<pgwire.PgConnection>[] = [];
-
-  constructor(
-    public options: NormalizedPostgresConnectionConfig,
-    public poolOptions: PgManagerOptions
-  ) {
-    super();
-    // The pool is lazy - no connections are opened until a query is performed.
-    this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
-    this.types = new PostgresTypeResolver(this.pool);
-  }
-
-  public get connectionTag() {
-    return this.options.tag;
-  }
-
-  /**
-   * Create a new replication connection.
-   */
-  async replicationConnection(): Promise<pgwire.PgConnection> {
-    const p = pgwire.connectPgWire(this.options, { type: 'replication', applicationName: getApplicationName() });
-    this.connectionPromises.push(p);
-    return await p;
-  }
-
-  /**
-   * @returns The Postgres server version in a parsed Semver instance
-   */
-  async getServerVersion(): Promise<semver.SemVer | null> {
-    return await getServerVersion(this.pool);
-  }
-
-  /**
-   * Create a new standard connection, used for initial snapshot.
-   *
-   * This connection must not be shared between multiple async contexts.
-   */
-  async snapshotConnection(): Promise<pgwire.PgConnection> {
-    const p = pgwire.connectPgWire(this.options, { type: 'standard', applicationName: getApplicationName() });
-    this.connectionPromises.push(p);
-    const connection = await p;
-
-    // Use an shorter timeout for snapshot connections.
-    // This is to detect broken connections early, instead of waiting
-    // for the full 6 minutes.
-    // This we are constantly using the connection, we don't need any
-    // custom keepalives.
-
-    (connection as any)._socket.setTimeout(SNAPSHOT_SOCKET_TIMEOUT);
-
-    // Disable statement timeout for snapshot queries.
-    // On Supabase, the default is 2 minutes.
-    await connection.query(`set session statement_timeout = 0`);
-
-    return connection;
-  }
-
-  async end(): Promise<void> {
-    for (let result of await Promise.allSettled([
-      this.pool.end(),
-      ...this.connectionPromises.map(async (promise) => {
-        // Wait for connection attempts to finish, but do not throw connection errors here
-        const connection = await promise.catch((_) => {});
-        return await connection?.end();
-      })
-    ])) {
-      // Throw the first error, if any
-      if (result.status == 'rejected') {
-        throw result.reason;
-      }
-    }
-    this.iterateListeners((listener) => {
-      listener.onEnded?.();
-    });
-  }
-
-  async destroy() {
-    this.pool.destroy();
-    for (let result of await Promise.allSettled([
-      ...this.connectionPromises.map(async (promise) => {
-        // Wait for connection attempts to finish, but do not throw connection errors here
-        const connection = await promise.catch((_) => {});
-        return connection?.destroy();
-      })
-    ])) {
-      // Throw the first error, if any
-      if (result.status == 'rejected') {
-        throw result.reason;
-      }
-    }
-    this.iterateListeners((listener) => {
-      listener.onEnded?.();
-    });
-  }
-}

package/src/replication/PgRelation.ts

@@ -1,41 +0,0 @@
-import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
-import { PgoutputRelation } from '@powersync/service-jpgwire';
-
-export type ReplicationIdentity = 'default' | 'nothing' | 'full' | 'index';
-
-export function getReplicaIdColumns(relation: PgoutputRelation): storage.ColumnDescriptor[] {
-  if (relation.replicaIdentity == 'nothing') {
-    return [];
-  } else {
-    return relation.columns
-      .filter((c) => (c.flags & 0b1) != 0)
-      .map((c) => ({ name: c.name, typeId: c.typeOid }) satisfies storage.ColumnDescriptor);
-  }
-}
-export function getRelId(source: PgoutputRelation): number {
-  // Source types are wrong here
-  const relId = (source as any).relationOid as number;
-  if (!relId) {
-    throw new ReplicationAssertionError(`No relation id found`);
-  }
-  return relId;
-}
-
-export function getPgOutputRelation(source: PgoutputRelation): storage.SourceEntityDescriptor {
-  return {
-    name: source.name,
-    schema: source.schema,
-    objectId: getRelId(source),
-    replicaIdColumns: getReplicaIdColumns(source)
-  } satisfies storage.SourceEntityDescriptor;
-}
-
-export function referencedColumnTypeIds(source: PgoutputRelation): number[] {
-  const oids = new Set<number>();
-  for (const column of source.columns) {
-    oids.add(column.typeOid);
-  }
-
-  return [...oids];
-}